Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace to gopkg.in/yaml with github.com/goccy/go-yaml (note) #13033

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion commands/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/alecthomas/chroma/v2/formatters/html"
"github.com/alecthomas/chroma/v2/styles"
"github.com/bep/simplecobra"
"github.com/goccy/go-yaml"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/docshelper"
"github.com/gohugoio/hugo/helpers"
Expand All @@ -35,7 +36,6 @@ import (
"github.com/gohugoio/hugo/parser"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"gopkg.in/yaml.v2"
)

func newGenCommand() *genCommand {
Expand Down
1 change: 0 additions & 1 deletion commands/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -1160,7 +1160,6 @@ func chmodFilter(dst, src os.FileInfo) bool {
}

func cleanErrorLog(content string) string {
content = strings.ReplaceAll(content, "\n", " ")
content = logReplacer.Replace(content)
content = logDuplicateTemplateExecuteRe.ReplaceAllString(content, "")
content = logDuplicateTemplateParseRe.ReplaceAllString(content, "")
Expand Down
19 changes: 9 additions & 10 deletions common/herrors/file_error.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,11 @@ func (fe *fileError) UpdateContent(r io.Reader, linematcher LineMatcherFn) FileE

fe.errorContext = ectx

if ectx.Position.LineNumber > 0 {
if ectx.Position.LineNumber > 0 && ectx.Position.LineNumber > fe.position.LineNumber {
fe.position.LineNumber = ectx.Position.LineNumber
}

if ectx.Position.ColumnNumber > 0 {
if ectx.Position.ColumnNumber > 0 && ectx.Position.ColumnNumber > fe.position.ColumnNumber {
fe.position.ColumnNumber = ectx.Position.ColumnNumber
}

Expand Down Expand Up @@ -177,6 +177,7 @@ func NewFileErrorFromName(err error, name string) FileError {
// Filetype is used to determine the Chroma lexer to use.
fileType, pos := extractFileTypePos(err)
pos.Filename = name

if fileType == "" {
_, fileType = paths.FileAndExtNoDelimiter(filepath.Clean(name))
}
Expand Down Expand Up @@ -234,7 +235,9 @@ func NewFileErrorFromFile(err error, filename string, fs afero.Fs, linematcher L
return NewFileErrorFromName(err, realFilename)
}
defer f.Close()
return NewFileErrorFromName(err, realFilename).UpdateContent(f, linematcher)
fe := NewFileErrorFromName(err, realFilename)
fe = fe.UpdateContent(f, linematcher)
return fe
}

func openFile(filename string, fs afero.Fs) (afero.File, string, error) {
Expand Down Expand Up @@ -302,13 +305,9 @@ func extractFileTypePos(err error) (string, text.Position) {
}

// Look in the error message for the line number.
for _, handle := range lineNumberExtractors {
lno, col := handle(err)
if lno > 0 {
pos.ColumnNumber = col
pos.LineNumber = lno
break
}
if lno, col := commonLineNumberExtractor(err); lno > 0 {
pos.ColumnNumber = col
pos.LineNumber = lno
}

if fileType == "" && pos.Filename != "" {
Expand Down
16 changes: 13 additions & 3 deletions common/herrors/line_number_extractors.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,27 @@ import (
)

var lineNumberExtractors = []lineNumberExtractor{
// YAML parse errors.
newLineNumberErrHandlerFromRegexp(`\[(\d+):(\d+)\]`),

// Template/shortcode parse errors
newLineNumberErrHandlerFromRegexp(`:(\d+):(\d*):`),
newLineNumberErrHandlerFromRegexp(`:(\d+):`),

// YAML parse errors
newLineNumberErrHandlerFromRegexp(`line (\d+):`),

// i18n bundle errors
newLineNumberErrHandlerFromRegexp(`\((\d+),\s(\d*)`),
}

func commonLineNumberExtractor(e error) (int, int) {
for _, handler := range lineNumberExtractors {
lno, col := handler(e)
if lno > 0 {
return lno, col
}
}
return 0, 0
}

type lineNumberExtractor func(e error) (int, int)

func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
Expand Down
31 changes: 31 additions & 0 deletions common/herrors/line_number_extractors_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package herrors

import (
"errors"
"testing"

qt "github.com/frankban/quicktest"
)

func TestCommonLineNumberExtractor(t *testing.T) {
t.Parallel()

c := qt.New(t)

lno, col := commonLineNumberExtractor(errors.New("[4:9] value is not allowed in this context"))
c.Assert(lno, qt.Equals, 4)
c.Assert(col, qt.Equals, 9)
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/swag v0.22.8 // indirect
github.com/goccy/go-yaml v1.15.3 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.8 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4
github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/goccy/go-yaml v1.15.3 h1:wQ4UwLFkgbSazdi+i9AVmZE3vKTktlNlI2kQqXo5L+I=
github.com/goccy/go-yaml v1.15.3/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ=
github.com/gohugoio/hashstructure v0.1.0 h1:kBSTMLMyTXbrJVAxaKI+wv30MMJJxn9Q8kfQtJaZ400=
Expand Down
4 changes: 2 additions & 2 deletions hugolib/frontmatter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Strings: {{ printf "%T" .Params.strings }} {{ range .Params.strings }}Strings: {

b.Build()

b.AssertFileContent("public/post/one/index.html", "Ints: []interface {} Int: 1 (int)|Int: 2 (int)|Int: 3 (int)|")
b.AssertFileContent("public/post/one/index.html", "Mixed: []interface {} Mixed: 1 (string)|Mixed: 2 (int)|Mixed: 3 (int)|")
b.AssertFileContent("public/post/one/index.html", "Ints: []interface {} Int: 1 (uint64)|Int: 2 (uint64)|Int: 3 (uint64)|")
b.AssertFileContent("public/post/one/index.html", "Mixed: []interface {} Mixed: 1 (string)|Mixed: 2 (uint64)|Mixed: 3 (uint64)|")
b.AssertFileContent("public/post/one/index.html", "Strings: []string Strings: 1 (string)|Strings: 2 (string)|Strings: 3 (string)|")
}
34 changes: 33 additions & 1 deletion hugolib/hugo_sites_build_errors_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ line 5
errors := herrors.UnwrapFileErrorsWithErrorContext(err)

b.Assert(errors, qt.HasLen, 3)
b.Assert(errors[0].Error(), qt.Contains, filepath.FromSlash(`"/content/_index.md:1:1": "/layouts/_default/_markup/render-heading.html:2:5": execute of template failed`))
b.Assert(errors[0].Error(), qt.Contains, filepath.FromSlash(`"/content/_index.md:2:5": "/layouts/_default/_markup/render-heading.html:2:5": execute of template failed`))
}

func TestErrorRenderHookCodeblock(t *testing.T) {
Expand Down Expand Up @@ -645,3 +645,35 @@ Home.
b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`/layouts/index.html:2:3`))
b.Assert(err.Error(), qt.Contains, `can't evaluate field ThisDoesNotExist`)
}

func TestErrorFrontmatterYAMLSyntax(t *testing.T) {
t.Parallel()

files := `
-- hugo.toml --
-- content/_index.md --





---
line1: 'value1'
x
line2: 'value2'
line3: 'value3'
---
`

b, err := TestE(t, files)

b.Assert(err, qt.Not(qt.IsNil))
b.Assert(err.Error(), qt.Contains, "> 3 |")
fe := herrors.UnwrapFileError(err)
b.Assert(fe, qt.Not(qt.IsNil))
pos := fe.Position()
b.Assert(pos.Filename, qt.Contains, filepath.FromSlash("content/_index.md"))
b.Assert(fe.ErrorContext(), qt.Not(qt.IsNil))
b.Assert(pos.LineNumber, qt.Equals, 9)
b.Assert(pos.ColumnNumber, qt.Equals, 1)
}
25 changes: 11 additions & 14 deletions hugolib/page__content.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,23 +283,20 @@ func (c *contentParseInfo) parseFrontMatter(it pageparser.Item, iter *pageparser
var err error
c.frontMatter, err = metadecoders.Default.UnmarshalToMap(it.Val(source), f)
if err != nil {
if fe, ok := err.(herrors.FileError); ok {
pos := fe.Position()
fe := herrors.UnwrapFileError(err)
if fe == nil {
fe = herrors.NewFileError(err)
}
pos := fe.Position()

// Offset the starting position of front matter.
offset := iter.LineNumber(source) - 1
if f == metadecoders.YAML {
offset -= 1
}
pos.LineNumber += offset
// Offset the starting position of front matter.
offset := iter.LineNumber(source) - 1

fe.UpdatePosition(pos)
fe.SetFilename("") // It will be set later.
pos.LineNumber += offset

return fe
} else {
return err
}
fe.UpdatePosition(pos)
fe.SetFilename("") // It will be set later.
return fe
}

return nil
Expand Down
2 changes: 1 addition & 1 deletion hugolib/pages_capture.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func (c *pagesCollector) Collect() (collectErr error) {
Handle: func(ctx context.Context, fi hugofs.FileMetaInfo) error {
numPages, numResources, err := c.m.AddFi(fi, c.buildConfig)
if err != nil {
return hugofs.AddFileInfoToError(err, fi, c.fs)
return hugofs.AddFileInfoToError(err, fi, c.h.SourceFs)
}
numFilesProcessedTotal.Add(1)
numPagesProcessedTotal.Add(numPages)
Expand Down
2 changes: 1 addition & 1 deletion langs/i18n/translationProvider.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import (

"github.com/gohugoio/hugo/common/paths"

yaml "github.com/goccy/go-yaml"
"github.com/gohugoio/hugo/common/herrors"
"golang.org/x/text/language"
yaml "gopkg.in/yaml.v2"

"github.com/gohugoio/go-i18n/v2/i18n"
"github.com/gohugoio/hugo/helpers"
Expand Down
4 changes: 1 addition & 3 deletions parser/frontmatter.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ import (

toml "github.com/pelletier/go-toml/v2"

yaml "gopkg.in/yaml.v2"

xml "github.com/clbanning/mxj/v2"
)

Expand All @@ -39,7 +37,7 @@ func InterfaceToConfig(in any, format metadecoders.Format, w io.Writer) error {

switch format {
case metadecoders.YAML:
b, err := yaml.Marshal(in)
b, err := metadecoders.MarshalYAML(in)
if err != nil {
return err
}
Expand Down
81 changes: 3 additions & 78 deletions parser/metadecoders/decoder.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,10 +28,10 @@ import (
"github.com/niklasfasching/go-org/org"

xml "github.com/clbanning/mxj/v2"
yaml "github.com/goccy/go-yaml"
toml "github.com/pelletier/go-toml/v2"
"github.com/spf13/afero"
"github.com/spf13/cast"
yaml "gopkg.in/yaml.v2"
)

// Decoder provides some configuration options for the decoders.
Expand Down Expand Up @@ -164,35 +164,7 @@ func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error {
case TOML:
err = toml.Unmarshal(data, v)
case YAML:
err = yaml.Unmarshal(data, v)
if err != nil {
return toFileError(f, data, fmt.Errorf("failed to unmarshal YAML: %w", err))
}

// To support boolean keys, the YAML package unmarshals maps to
// map[interface{}]interface{}. Here we recurse through the result
// and change all maps to map[string]interface{} like we would've
// gotten from `json`.
var ptr any
switch vv := v.(type) {
case *map[string]any:
ptr = *vv
case *any:
ptr = *vv
default:
// Not a map.
}

if ptr != nil {
if mm, changed := stringifyMapKeys(ptr); changed {
switch vv := v.(type) {
case *map[string]any:
*vv = mm.(map[string]any)
case *any:
*vv = mm
}
}
}
return yaml.Unmarshal(data, v)
case CSV:
return d.unmarshalCSV(data, v)

Expand Down Expand Up @@ -269,50 +241,3 @@ func (d Decoder) unmarshalORG(data []byte, v any) error {
func toFileError(f Format, data []byte, err error) error {
return herrors.NewFileErrorFromName(err, fmt.Sprintf("_stream.%s", f)).UpdateContent(bytes.NewReader(data), nil)
}

// stringifyMapKeys recurses into in and changes all instances of
// map[interface{}]interface{} to map[string]interface{}. This is useful to
// work around the impedance mismatch between JSON and YAML unmarshaling that's
// described here: https://github.com/go-yaml/yaml/issues/139
//
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
func stringifyMapKeys(in any) (any, bool) {
switch in := in.(type) {
case []any:
for i, v := range in {
if vv, replaced := stringifyMapKeys(v); replaced {
in[i] = vv
}
}
case map[string]any:
for k, v := range in {
if vv, changed := stringifyMapKeys(v); changed {
in[k] = vv
}
}
case map[any]any:
res := make(map[string]any)
var (
ok bool
err error
)
for k, v := range in {
var ks string

if ks, ok = k.(string); !ok {
ks, err = cast.ToStringE(k)
if err != nil {
ks = fmt.Sprintf("%v", k)
}
}
if vv, replaced := stringifyMapKeys(v); replaced {
res[ks] = vv
} else {
res[ks] = v
}
}
return res, true
}

return nil, false
}
Loading
Loading