resources: Add FromOpts for more effective resource creation

E.g. when the targetPath already contains a hash or if the resource content is expensive to create.
This commit is contained in:
Bjørn Erik Pedersen 2025-01-03 18:36:44 +01:00
parent d913f46a8b
commit 723e3f4342
2 changed files with 81 additions and 7 deletions

View file

@ -38,6 +38,19 @@ func XXHashFromReader(r io.Reader) (uint64, int64, error) {
return h.Sum64(), size, nil return h.Sum64(), size, nil
} }
// XxHashFromReaderHexEncoded calculates the xxHash for the given reader
// and returns the hash as a hex encoded string.
func XxHashFromReaderHexEncoded(r io.Reader) (string, error) {
h := getXxHashReadFrom()
defer putXxHashReadFrom(h)
_, err := io.Copy(h, r)
if err != nil {
return "", err
}
hash := h.Sum(nil)
return hex.EncodeToString(hash), nil
}
// XXHashFromString calculates the xxHash for the given string. // XXHashFromString calculates the xxHash for the given string.
func XXHashFromString(s string) (uint64, error) { func XXHashFromString(s string) (uint64, error) {
h := xxhash.New() h := xxhash.New()

View file

@ -218,22 +218,83 @@ func (c *Client) match(name, pattern string, matchFunc func(r resource.Resource)
}) })
} }
// FromString creates a new Resource from a string with the given relative target path. type Options struct {
// TODO(bep) see #10912; we currently emit a warning for this config scenario. // The target path relative to the publish directory.
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { // Unix style path, i.e. "images/logo.png".
targetPath = path.Clean(targetPath) TargetPath string
key := dynacache.CleanKey(targetPath) + hashing.MD5FromStringHexEncoded(content)
// Whether the TargetPath has a hash in it which will change if the resource changes.
// If not, we will calculate a hash from the content.
TargetPathHasHash bool
// The content to create the Resource from.
CreateContent func() (func() (hugio.ReadSeekCloser, error), error)
}
// FromOpts creates a new Resource from the given Options.
// Make sure to set optis.TargetPathHasHash if the TargetPath already contains a hash,
// as this avoids the need to calculate it.
// To create a new ReadSeekCloser from a string, use hugio.NewReadSeekerNoOpCloserFromString,
// or hugio.NewReadSeekerNoOpCloserFromBytes for a byte slice.
// See FromString.
func (c *Client) FromOpts(opts Options) (resource.Resource, error) {
opts.TargetPath = path.Clean(opts.TargetPath)
var hash string
var newReadSeeker func() (hugio.ReadSeekCloser, error) = nil
if !opts.TargetPathHasHash {
var err error
newReadSeeker, err = opts.CreateContent()
if err != nil {
return nil, err
}
if err := func() error {
r, err := newReadSeeker()
if err != nil {
return err
}
defer r.Close()
hash, err = hashing.XxHashFromReaderHexEncoded(r)
if err != nil {
return err
}
return nil
}(); err != nil {
return nil, err
}
}
key := dynacache.CleanKey(opts.TargetPath) + hash
r, err := c.rs.ResourceCache.GetOrCreate(key, func() (resource.Resource, error) { r, err := c.rs.ResourceCache.GetOrCreate(key, func() (resource.Resource, error) {
if newReadSeeker == nil {
var err error
newReadSeeker, err = opts.CreateContent()
if err != nil {
return nil, err
}
}
return c.rs.NewResource( return c.rs.NewResource(
resources.ResourceSourceDescriptor{ resources.ResourceSourceDescriptor{
LazyPublish: true, LazyPublish: true,
GroupIdentity: identity.Anonymous, // All usage of this resource are tracked via its string content. GroupIdentity: identity.Anonymous, // All usage of this resource are tracked via its string content.
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(content), nil return newReadSeeker()
}, },
TargetPath: targetPath, TargetPath: opts.TargetPath,
}) })
}) })
return r, err return r, err
} }
// FromString creates a new Resource from a string with the given relative target path.
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
return c.FromOpts(Options{
TargetPath: targetPath,
CreateContent: func() (func() (hugio.ReadSeekCloser, error), error) {
return func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(content), nil
}, nil
},
})
}