From 002f409e676111ab64228e7d45a6766062e9fdff Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Jul 2024 12:06:08 +0800 Subject: [PATCH] Optimize SUI variable parsing logic --- sui/core/build.go | 2 +- sui/core/context.go | 8 +- sui/core/data.go | 56 ++--- sui/core/jit.go | 42 ++-- sui/core/parser.go | 6 +- sui/core/token.go | 165 ++++++++++++++ sui/core/token_test.go | 476 +++++++++++++++++++++++++++++++++++++++++ sui/core/translate.go | 2 +- 8 files changed, 710 insertions(+), 47 deletions(-) create mode 100644 sui/core/token.go create mode 100644 sui/core/token_test.go diff --git a/sui/core/build.go b/sui/core/build.go index b990208f..83ad12e9 100644 --- a/sui/core/build.go +++ b/sui/core/build.go @@ -338,7 +338,7 @@ func (page *Page) parseProps(from *goquery.Selection, to *goquery.Selection, ext } trans := from.AttrOr(fmt.Sprintf("s:trans-attr-%s", attr.Key), "") - exp := stmtRe.Match([]byte(attr.Val)) + exp := dataTokens.MatchString(attr.Val) prop := PageProp{Key: attr.Key, Val: attr.Val, Trans: trans, Exp: exp} page.props[attr.Key] = prop to.SetAttr(attr.Key, attr.Val) diff --git a/sui/core/context.go b/sui/core/context.go index 4c6c9fb9..7e8fd02a 100644 --- a/sui/core/context.go +++ b/sui/core/context.go @@ -78,8 +78,8 @@ func (globalCtx *GlobalBuildContext) GetJitComponents() []string { } func (ctx *BuildContext) addJitComponent(name string) { - name = stmtRe.ReplaceAllString(name, "*") - name = propRe.ReplaceAllString(name, "*") + name = dataTokens.ReplaceAllString(name, "*") + name = propTokens.ReplaceAllString(name, "*") ctx.jitComponents[name] = true if ctx.global != nil { ctx.global.jitComponents[name] = true @@ -87,7 +87,7 @@ func (ctx *BuildContext) addJitComponent(name string) { } func (ctx *BuildContext) isJitComponent(name string) bool { - hasStmt := stmtRe.MatchString(name) - hasProp := propRe.MatchString(name) + hasStmt := dataTokens.MatchString(name) + hasProp := propTokens.MatchString(name) return hasStmt || hasProp } diff --git a/sui/core/data.go b/sui/core/data.go index 17aab80e..8a761d26 100644 --- a/sui/core/data.go +++ b/sui/core/data.go @@ -16,9 +16,9 @@ import ( ) // If set the map value, should keep the space at the end of the statement -var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`) -var propRe = regexp.MustCompile(`\[\{([\s\S]*?)\}\]`) // [{ xxx }] will be deprecated -var propNewRe = regexp.MustCompile(`\{%([\s\S]*)?%\}`) // {% xxx %} +// var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`) +// var propRe = regexp.MustCompile(`\[\{([\s\S]*?)\}\]`) // [{ xxx }] will be deprecated +// var propNewRe = regexp.MustCompile(`\{%([\s\S]*)?%\}`) // {% xxx %} var propVarNameRe = regexp.MustCompile(`(?:\$props\.)?(?:$begin:math:display$'([^']+)'$end:math:display$|(\w+))`) // Data data for the template @@ -80,18 +80,18 @@ func (data Data) Hash() string { // New create a new expression func (data Data) New(stmt string) (*vm.Program, error) { - stmt = stmtRe.ReplaceAllStringFunc(stmt, func(stmt string) string { - matches := stmtRe.FindStringSubmatch(stmt) + stmt = dataTokens.ReplaceAllStringFunc(stmt, func(stmt string) string { + matches := dataTokens.FindAllStringSubmatch(stmt, -1) if len(matches) > 0 { - stmt = strings.ReplaceAll(stmt, matches[0], matches[1]) + stmt = strings.ReplaceAll(stmt, matches[0][0], matches[0][1]) } return stmt }) - stmt = propRe.ReplaceAllStringFunc(stmt, func(stmt string) string { - matches := propRe.FindStringSubmatch(stmt) + stmt = propTokens.ReplaceAllStringFunc(stmt, func(stmt string) string { + matches := propTokens.FindAllStringSubmatch(stmt, -1) if len(matches) > 0 { - stmt = strings.ReplaceAll(stmt, matches[0], matches[1]) + stmt = strings.ReplaceAll(stmt, matches[0][0], matches[0][1]) } return stmt }) @@ -122,6 +122,18 @@ func (data Data) Exec(stmt string) (interface{}, []Identifier, error) { return res, v.Identifiers, nil } +// Identifiers get the identifiers for the statement +func (data Data) Identifiers(stmt string) ([]Identifier, error) { + program, err := data.New(stmt) + if err != nil { + return nil, err + } + node := program.Node() + v := &Visitor{} + ast.Walk(&node, v) + return v.Identifiers, nil +} + // ExecString exec statement for the template func (data Data) ExecString(stmt string) StringValue { @@ -163,13 +175,13 @@ func (data Data) ExecString(stmt string) StringValue { // Replace replace the statement func (data Data) Replace(value string) (string, []StringValue) { - return data.ReplaceUse(stmtRe, value) + return data.ReplaceUse(dataTokens, value) } // ReplaceUse replace the statement use the regexp -func (data Data) ReplaceUse(re *regexp.Regexp, value string) (string, []StringValue) { +func (data Data) ReplaceUse(tokens Tokens, value string) (string, []StringValue) { values := []StringValue{} - res := re.ReplaceAllStringFunc(value, func(stmt string) string { + res := tokens.ReplaceAllStringFunc(value, func(stmt string) string { v := data.ExecString(stmt) values = append(values, v) return v.Value @@ -179,14 +191,14 @@ func (data Data) ReplaceUse(re *regexp.Regexp, value string) (string, []StringVa // ReplaceSelection replace the statement in the selection func (data Data) ReplaceSelection(sel *goquery.Selection) []StringValue { - return data.ReplaceSelectionUse(stmtRe, sel) + return data.ReplaceSelectionUse(dataTokens, sel) } // ReplaceSelectionUse replace the statement in the selection use the regexp -func (data Data) ReplaceSelectionUse(re *regexp.Regexp, sel *goquery.Selection) []StringValue { +func (data Data) ReplaceSelectionUse(tokens Tokens, sel *goquery.Selection) []StringValue { res := []StringValue{} for _, node := range sel.Nodes { - values := data.replaceNodeUse(re, node) + values := data.replaceNodeUse(tokens, node) if len(values) > 0 { res = append(res, values...) } @@ -194,11 +206,11 @@ func (data Data) ReplaceSelectionUse(re *regexp.Regexp, sel *goquery.Selection) return res } -func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValue { +func (data Data) replaceNodeUse(tokens Tokens, node *html.Node) []StringValue { res := []StringValue{} switch node.Type { case html.TextNode: - v, values := data.ReplaceUse(re, node.Data) + v, values := data.ReplaceUse(tokens, node.Data) node.Data = v if len(values) > 0 { res = append(res, values...) @@ -212,7 +224,7 @@ func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValu continue } - v, values := data.ReplaceUse(re, node.Attr[i].Val) + v, values := data.ReplaceUse(tokens, node.Attr[i].Val) node.Attr[i].Val = v if len(values) > 0 { res = append(res, values...) @@ -220,7 +232,7 @@ func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValu } for c := node.FirstChild; c != nil; c = c.NextSibling { - values := data.replaceNodeUse(re, c) + values := data.replaceNodeUse(tokens, c) if len(values) > 0 { res = append(res, values...) } @@ -288,11 +300,7 @@ func _process(args ...any) (interface{}, error) { // PropFindAllStringSubmatch find all string submatch func PropFindAllStringSubmatch(value string) [][]string { - matched := propNewRe.FindAllStringSubmatch(value, -1) - oldVersion := propRe.FindAllStringSubmatch(value, -1) // will be deprecated - if len(oldVersion) > 0 { - matched = append(matched, oldVersion...) - } + matched := propTokens.FindAllStringSubmatch(value, -1) return matched } diff --git a/sui/core/jit.go b/sui/core/jit.go index a0673f3f..26787220 100644 --- a/sui/core/jit.go +++ b/sui/core/jit.go @@ -69,7 +69,17 @@ func (parser *TemplateParser) parseJitComponent(sel *goquery.Selection) { func (parser *TemplateParser) RenderComponent(comp *JitComponent, props map[string]interface{}, slots *goquery.Selection, children *goquery.Selection) (string, string, error) { html := comp.html + fmt.Println("---", comp.route, "----------------") + fmt.Println(html) + fmt.Println("-------------------------------") + fmt.Println() html = replaceRandVar(html, Data(props)) + fmt.Println("---- after replaceRandVar ----------------") + fmt.Println() + fmt.Println(html) + fmt.Println("-------------------------------") + fmt.Println() + option := *parser.option option.Route = comp.route compParser := NewTemplateParser(parser.data, &option) @@ -115,11 +125,7 @@ func (parser *TemplateParser) RenderComponent(comp *JitComponent, props map[stri compParser.locale = locale compParser.BindEvent(root, ns, cn) - compParser.parseNode(root.Get(0)) - for sel, nodes := range compParser.replace { - sel.ReplaceWithNodes(nodes...) - delete(parser.replace, sel) - } + compParser.RenderSelection(root) if compParser.scripts != nil { for _, script := range compParser.scripts { @@ -312,7 +318,7 @@ func (parser *TemplateParser) componentFile(sel *goquery.Selection, props map[st } data := Data{"$props": props} - route, _ = data.ReplaceUse(slotRe, route) + route, _ = data.ReplaceUse(dataTokens, route) route, _ = parser.data.Replace(route) file := filepath.Join(string(os.PathSeparator), "public", parser.option.Root, route+".jit") return file, route, nil @@ -431,16 +437,24 @@ func readComponent(route string, file string) (*JitComponent, error) { func replaceRandVar(value string, data Data) string { - value = propNewRe.ReplaceAllStringFunc(value, func(exp string) string { - exp = strings.TrimPrefix(exp, "{%") - exp = strings.TrimSuffix(exp, "%}") - res := data.ExecString(fmt.Sprintf("{{ %s }}", exp)) - return res.Value - }) + return propTokens.ReplaceAllStringFunc(value, func(exp string) string { + if strings.HasPrefix(exp, "[{") && strings.HasSuffix(exp, "}]") { + matches := propTokens.FindAllStringSubmatch(exp, -1) + if len(matches) > 0 { + identifiers, err := data.Identifiers(matches[0][1]) + if err != nil { + return err.Error() + } + for _, identifier := range identifiers { + exp = strings.ReplaceAll(exp, identifier.Value, strings.ReplaceAll(identifier.Value, "$props.", "")) + } + } - data = Data{"$props": data} - return slotRe.ReplaceAllStringFunc(value, func(exp string) string { + } res := data.ExecString(exp) + if res.Error != nil { + return res.Error.Error() + } return res.Value }) } diff --git a/sui/core/parser.go b/sui/core/parser.go index dd33a2e7..df18eb51 100644 --- a/sui/core/parser.go +++ b/sui/core/parser.go @@ -371,7 +371,7 @@ func (parser *TemplateParser) transElementNode(sel *goquery.Selection) { // Escape the text func (parser *TemplateParser) escapeText(content string) string { - matches := stmtRe.FindAllStringSubmatch(content, -1) + matches := dataTokens.FindAllStringSubmatch(content, -1) newContent := content for _, match := range matches { text := strings.TrimSpace(match[1]) @@ -419,7 +419,7 @@ func (parser *TemplateParser) transNode(key string, message string) string { func (parser *TemplateParser) transText(content string, keys []string) string { - matches := stmtRe.FindAllStringSubmatch(content, -1) + matches := dataTokens.FindAllStringSubmatch(content, -1) newContent := content for _, match := range matches { text := strings.TrimSpace(match[1]) @@ -483,7 +483,7 @@ func (parser *TemplateParser) setStatementNode(sel *goquery.Selection) { } valueExp := sel.AttrOr("value", "") - if stmtRe.MatchString(valueExp) { + if dataTokens.MatchString(valueExp) { val, _, err := parser.data.Exec(valueExp) if err != nil { log.Warn("Set %s: %s", valueExp, err) diff --git a/sui/core/token.go b/sui/core/token.go new file mode 100644 index 00000000..360d1668 --- /dev/null +++ b/sui/core/token.go @@ -0,0 +1,165 @@ +package core + +import ( + "strings" +) + +var propTokens = Tokens{ + {start: "{%", end: "%}"}, // {% xxx %} + {start: "[{", end: "}]"}, // [{ xxx }] +} + +var dataTokens = Tokens{ + {start: "{{", end: "}}"}, // {{ xxx }} +} + +// Token the token +type Token struct { + start string + end string +} + +// Tokens the tokens +type Tokens []Token + +// FindStringSubmatch returns a slice of strings holding the text of the +// leftmost match of the regular expression in s and the matches, if any, of +// its subexpressions, as defined by the 'Submatch' description in the +// package comment. +// A return value of nil indicates no match. +func (tokens Tokens) FindStringSubmatch(s string) []string { + matches := []string{} + for _, token := range tokens { + startLen := len(token.start) + endLen := len(token.end) + stack := 0 + for i := 0; i <= len(s)-startLen; i++ { + if s[i:i+startLen] == token.start { + stack++ + if stack == 1 { + for j := i + startLen; j <= len(s)-endLen; j++ { + if s[j:j+endLen] == token.end { + stack-- + if stack == 0 { + matches = append(matches, s[i:j+endLen]) + i = j + endLen - 1 + break + } + } else if s[j:j+startLen] == token.start { + stack++ + } + } + } + } + } + } + if len(matches) == 0 { + return nil + } + return matches +} + +// FindAllStringSubmatch is the 'All' version of FindStringSubmatch; it +// returns a slice of all successive matches of the expression, as defined by +// the 'All' description in the package comment. +// A return value of nil indicates no match. +func (tokens Tokens) FindAllStringSubmatch(s string, n int) [][]string { + matches := [][]string{} + for _, token := range tokens { + startLen := len(token.start) + endLen := len(token.end) + stack := 0 + for i := 0; i <= len(s)-startLen; i++ { + if s[i:i+startLen] == token.start { + stack++ + if stack == 1 { + for j := i + startLen; j <= len(s)-endLen; j++ { + if s[j:j+endLen] == token.end { + stack-- + if stack == 0 { + matches = append(matches, []string{s[i : j+endLen], strings.TrimSpace(s[i+startLen : j])}) + i = j + endLen - 1 + break + } + } else if s[j:j+startLen] == token.start { + stack++ + } + } + } + } + } + } + if len(matches) == 0 { + return nil + } + return matches +} + +// MatchString reports whether the string s +// contains any match of the regular expression re. +func (tokens Tokens) MatchString(s string) bool { + for _, token := range tokens { + startLen := len(token.start) + endLen := len(token.end) + stack := 0 + for i := 0; i <= len(s)-startLen; i++ { + if s[i:i+startLen] == token.start { + stack++ + if stack == 1 { + for j := i + startLen; j <= len(s)-endLen; j++ { + if s[j:j+endLen] == token.end { + stack-- + if stack == 0 { + return true + } + } else if s[j:j+startLen] == token.start { + stack++ + } + } + } + } + } + } + return false +} + +// ReplaceAllStringFunc returns a copy of src in which all matches of the +// Regexp have been replaced by the return value of function repl applied +// to the matched substring. The replacement returned by repl is substituted +// directly, without using Expand. +func (tokens Tokens) ReplaceAllStringFunc(src string, repl func(string) string) string { + for _, token := range tokens { + startLen := len(token.start) + endLen := len(token.end) + stack := 0 + i := 0 + for i <= len(src)-startLen { + if i+startLen <= len(src) && src[i:i+startLen] == token.start { + stack++ + if stack == 1 { + for j := i + startLen; j <= len(src)-endLen; j++ { + if j+endLen <= len(src) && src[j:j+endLen] == token.end { + stack-- + if stack == 0 { + match := src[i : j+endLen] + replacement := repl(match) + src = src[:i] + replacement + src[j+endLen:] + i = i + len(replacement) - 1 + break + } + } else if j+startLen <= len(src) && src[j:j+startLen] == token.start { + stack++ + } + } + } + } + i++ + } + } + return src +} + +// ReplaceAllString returns a copy of src in which all matches of the +func (tokens Tokens) ReplaceAllString(src, repl string) string { + return tokens.ReplaceAllStringFunc(src, func(string) string { return repl }) +} diff --git a/sui/core/token_test.go b/sui/core/token_test.go new file mode 100644 index 00000000..2fc55f59 --- /dev/null +++ b/sui/core/token_test.go @@ -0,0 +1,476 @@ +package core + +import ( + "testing" +) + +func TestTokensFindStringSubmatch(t *testing.T) { + propTokens := Tokens{ + {start: "{%", end: "%}"}, + {start: "[{", end: "}]"}, + } + + dataTokens := Tokens{ + {start: "{{", end: "}}"}, + } + + tests := []struct { + name string + tokens Tokens + input string + expected []string + }{ + { + name: "Match with propTokens - {% xxx %}", + tokens: propTokens, + input: "This is a test string {% match %} with tokens.", + expected: []string{"{% match %}"}, + }, + { + name: "Match with propTokens - [{ xxx }]", + tokens: propTokens, + input: "This is a test string [{ match }] with tokens.", + expected: []string{"[{ match }]"}, + }, + { + name: "Match with dataTokens - {{ xxx }}", + tokens: dataTokens, + input: "This is a test string {{ match }} with tokens.", + expected: []string{"{{ match }}"}, + }, + { + name: "No match", + tokens: propTokens, + input: "This string has no matching tokens.", + expected: nil, + }, + { + name: "Partial match start token", + tokens: propTokens, + input: "This string has a partial {% match.", + expected: nil, + }, + { + name: "Nested tokens", + tokens: propTokens, + input: "This string has {% outer {% inner %} outer %} tokens.", + expected: []string{"{% outer {% inner %} outer %}"}, + }, + { + name: "Multiple matches", + tokens: propTokens, + input: "This string has {% first %} and [{ second }] tokens.", + expected: []string{"{% first %}", "[{ second }]"}, + }, + { + name: "Empty input", + tokens: propTokens, + input: "", + expected: nil, + }, + { + name: "Adjacent tokens", + tokens: propTokens, + input: "{% first %}{% second %}", + expected: []string{"{% first %}", "{% second %}"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tokens.FindStringSubmatch(tt.input) + if !stringsEqual(result, tt.expected) { + t.Errorf("%s: Expected %v, got %v", tt.input, tt.expected, result) + } + }) + } +} + +func TestTokensFindAllStringSubmatch(t *testing.T) { + propTokens := Tokens{ + {start: "{%", end: "%}"}, + {start: "[{", end: "}]"}, + } + + dataTokens := Tokens{ + {start: "{{", end: "}}"}, + } + + tests := []struct { + name string + tokens Tokens + input string + n int + expected [][]string + }{ + { + name: "Single match with propTokens - {% xxx %}", + tokens: propTokens, + input: "This is a test string {% match %} with tokens.", + n: -1, // -1 indicates no limit + expected: [][]string{{"{% match %}", "match"}}, + }, + { + name: "Multiple matches with propTokens", + tokens: propTokens, + input: "This string has {% first %} and [{ second }] tokens.", + n: -1, + expected: [][]string{{"{% first %}", "first"}, {"[{ second }]", "second"}}, + }, + { + name: "Nested tokens with propTokens", + tokens: propTokens, + input: "This string has {% outer {% inner %} outer %} tokens.", + n: -1, + expected: [][]string{{"{% outer {% inner %} outer %}", "outer {% inner %} outer"}}, + }, + { + name: "Match with dataTokens - {{ xxx }}", + tokens: dataTokens, + input: "This is a test string {{ match }} with tokens.", + n: -1, + expected: [][]string{{"{{ match }}", "match"}}, + }, + { + name: "Multiple matches with dataTokens", + tokens: dataTokens, + input: "This string has {{ first }} and {{ second }} tokens.", + n: -1, + expected: [][]string{{"{{ first }}", "first"}, {"{{ second }}", "second"}}, + }, + { + name: "No match", + tokens: propTokens, + input: "This string has no matching tokens.", + n: -1, + expected: nil, + }, + { + name: "Partial match start token", + tokens: propTokens, + input: "This string has a partial {% match.", + n: -1, + expected: nil, + }, + { + name: "Empty input", + tokens: propTokens, + input: "", + n: -1, + expected: nil, + }, + { + name: "Limit matches", + tokens: propTokens, + input: "{% first %}{% second %}{% third %}", + n: 2, + expected: [][]string{{"{% first %}", "first"}, {"{% second %}", "second"}, {"{% third %}", "third"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tokens.FindAllStringSubmatch(tt.input, tt.n) + if !string2dEqual(result, tt.expected) { + t.Errorf("%s: expected %v, got %v", tt.input, tt.expected, result) + } + }) + } +} + +func TestTokensMatchString(t *testing.T) { + propTokens := Tokens{ + {start: "{%", end: "%}"}, + {start: "[{", end: "}]"}, + } + + dataTokens := Tokens{ + {start: "{{", end: "}}"}, + } + + tests := []struct { + name string + tokens Tokens + input string + expected bool + }{ + { + name: "Single match with propTokens - {% xxx %}", + tokens: propTokens, + input: "This is a test string {% match %} with tokens.", + expected: true, + }, + { + name: "Multiple matches with propTokens", + tokens: propTokens, + input: "This string has {% first %} and [{ second }] tokens.", + expected: true, + }, + { + name: "Nested tokens with propTokens", + tokens: propTokens, + input: "This string has {% outer {% inner %} outer %} tokens.", + expected: true, + }, + { + name: "Match with dataTokens - {{ xxx }}", + tokens: dataTokens, + input: "This is a test string {{ match }} with tokens.", + expected: true, + }, + { + name: "Multiple matches with dataTokens", + tokens: dataTokens, + input: "This string has {{ first }} and {{ second }} tokens.", + expected: true, + }, + { + name: "No match", + tokens: propTokens, + input: "This string has no matching tokens.", + expected: false, + }, + { + name: "Partial match start token", + tokens: propTokens, + input: "This string has a partial {% match.", + expected: false, + }, + { + name: "Empty input", + tokens: propTokens, + input: "", + expected: false, + }, + { + name: "Token at the end", + tokens: propTokens, + input: "Token at the end {% last %}", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tokens.MatchString(tt.input) + if result != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestTokensReplaceAllStringFunc(t *testing.T) { + propTokens := Tokens{ + {start: "{%", end: "%}"}, + {start: "[{", end: "}]"}, + } + + dataTokens := Tokens{ + {start: "{{", end: "}}"}, + } + + tests := []struct { + name string + tokens Tokens + input string + repl func(string) string + expected string + }{ + { + name: "Single replacement with propTokens", + tokens: propTokens, + input: "This is a test string {% replace this %}.", + repl: func(s string) string { return "[REPLACED]" }, + expected: "This is a test string [REPLACED].", + }, + { + name: "Multiple replacements with propTokens", + tokens: propTokens, + input: "This string {% first %} and [{ second }] will be replaced.", + repl: func(s string) string { return "REPLACED" }, + expected: "This string REPLACED and REPLACED will be replaced.", + }, + { + name: "Nested replacements with propTokens", + tokens: propTokens, + input: "This string has {% outer {% inner %} outer %} tokens.", + repl: func(s string) string { return "[NESTED]" }, + expected: "This string has [NESTED] tokens.", + }, + { + name: "Replacement with dataTokens", + tokens: dataTokens, + input: "This is a test string {{ replace this }}.", + repl: func(s string) string { return "REPLACED" }, + expected: "This is a test string REPLACED.", + }, + { + name: "Multiple replacements with dataTokens", + tokens: dataTokens, + input: "This string has {{ first }} and {{ second }} tokens.", + repl: func(s string) string { return "REPLACED" }, + expected: "This string has REPLACED and REPLACED tokens.", + }, + { + name: "No match", + tokens: propTokens, + input: "This string has no matching tokens.", + repl: func(s string) string { return "REPLACED" }, + expected: "This string has no matching tokens.", + }, + { + name: "Partial match start token", + tokens: propTokens, + input: "This string has a partial {% match.", + repl: func(s string) string { return "REPLACED" }, + expected: "This string has a partial {% match.", + }, + { + name: "Empty input", + tokens: propTokens, + input: "", + repl: func(s string) string { return "REPLACED" }, + expected: "", + }, + { + name: "Token at the end", + tokens: propTokens, + input: "Token at the end {% last %}", + repl: func(s string) string { return "REPLACED" }, + expected: "Token at the end REPLACED", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tokens.ReplaceAllStringFunc(tt.input, tt.repl) + if result != tt.expected { + t.Errorf("expected %q, got %q", tt.expected, result) + } + }) + } + +} + +func TestTokensReplaceAllString(t *testing.T) { + + propTokens := Tokens{ + {start: "{%", end: "%}"}, + {start: "[{", end: "}]"}, + } + + dataTokens := Tokens{ + {start: "{{", end: "}}"}, + } + + tests := []struct { + name string + tokens Tokens + input string + repl string + expected string + }{ + { + name: "Single replacement with propTokens", + tokens: propTokens, + input: "This is a test string {% replace this %}.", + repl: "[REPLACED]", + expected: "This is a test string [REPLACED].", + }, + { + name: "Multiple replacements with propTokens", + tokens: propTokens, + input: "This string {% first %} and [{ second }] will be replaced.", + repl: "REPLACED", + expected: "This string REPLACED and REPLACED will be replaced.", + }, + { + name: "Nested replacements with propTokens", + tokens: propTokens, + input: "This string has {% outer {% inner %} outer %} tokens.", + repl: "[NESTED]", + expected: "This string has [NESTED] tokens.", + }, + { + name: "Replacement with dataTokens", + tokens: dataTokens, + input: "This is a test string {{ replace this }}.", + repl: "REPLACED", + expected: "This is a test string REPLACED.", + }, + { + name: "Multiple replacements with dataTokens", + tokens: dataTokens, + input: "This string has {{ first }} and {{ second }} tokens.", + repl: "REPLACED", + expected: "This string has REPLACED and REPLACED tokens.", + }, + { + name: "No match", + tokens: propTokens, + input: "This string has no matching tokens.", + repl: "REPLACED", + expected: "This string has no matching tokens.", + }, + { + name: "Partial match start token", + tokens: propTokens, + input: "This string has a partial {% match.", + repl: "REPLACED", + expected: "This string has a partial {% match.", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tokens.ReplaceAllString(tt.input, tt.repl) + if result != tt.expected { + t.Errorf("expected %q, got %q", tt.expected, result) + } + }) + } + +} + +// Helper function to check equality of two slices of slices of strings +func string2dEqual(a, b [][]string) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + if len(a) != len(b) { + return false + } + for i := range a { + if len(a[i]) != len(b[i]) { + return false + } + for j := range a[i] { + if a[i][j] != b[i][j] { + return false + } + } + } + return true +} + +func stringsEqual(a, b []string) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/sui/core/translate.go b/sui/core/translate.go index 327ac486..39cbf058 100644 --- a/sui/core/translate.go +++ b/sui/core/translate.go @@ -160,7 +160,7 @@ func (page *Page) translateNode(node *html.Node) ([]Translation, error) { func (page *Page) translateText(text string, transType string) ([]Translation, []string, error) { translations := []Translation{} - matches := stmtRe.FindAllStringSubmatch(text, -1) + matches := dataTokens.FindAllStringSubmatch(text, -1) keys := []string{} for _, match := range matches { text := strings.TrimSpace(match[1])