From b21a55febfad30070772563eaf725300299d2c6c Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Wed, 26 Apr 2023 21:20:22 +0400 Subject: [PATCH] app/vmalert: add support of recursive path globs for rules and templates (#4148) Supports using `**` for `-rule` and `-rule.templates`: `dir/**/*.tpl` loads contents of dir and all subdirectories recursively. See: #4041 Signed-off-by: Zakhar Bessarab Co-authored-by: Artem Navoiev Co-authored-by: Nikolay --- app/vmalert/README.md | 4 +- app/vmalert/config/fslocal/fslocal.go | 7 +- app/vmalert/main.go | 8 +- app/vmalert/templates/template.go | 7 +- docs/CHANGELOG.md | 1 + docs/vmalert.md | 4 +- go.mod | 2 + go.sum | 2 + .../bmatcuk/doublestar/v4/.codecov.yml | 10 + .../bmatcuk/doublestar/v4/.gitignore | 32 ++ .../github.com/bmatcuk/doublestar/v4/LICENSE | 22 + .../bmatcuk/doublestar/v4/README.md | 404 +++++++++++++++ .../bmatcuk/doublestar/v4/UPGRADING.md | 63 +++ .../bmatcuk/doublestar/v4/doublestar.go | 13 + .../github.com/bmatcuk/doublestar/v4/glob.go | 473 ++++++++++++++++++ .../bmatcuk/doublestar/v4/globoptions.go | 144 ++++++ .../bmatcuk/doublestar/v4/globwalk.go | 414 +++++++++++++++ .../github.com/bmatcuk/doublestar/v4/match.go | 376 ++++++++++++++ .../github.com/bmatcuk/doublestar/v4/utils.go | 147 ++++++ .../bmatcuk/doublestar/v4/validate.go | 82 +++ vendor/modules.txt | 3 + 21 files changed, 2208 insertions(+), 10 deletions(-) create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/.codecov.yml create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/.gitignore create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/LICENSE create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/README.md create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/UPGRADING.md create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/doublestar.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/glob.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/globoptions.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/globwalk.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/match.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/utils.go create mode 100644 vendor/github.com/bmatcuk/doublestar/v4/validate.go diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 205bb0b72..1e9c489e4 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -1156,7 +1156,8 @@ The shortlist of configuration flags is the following: Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules - -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/**/*.yaml". Includes to all .yaml files in "dir" folder and it's subfolders recursively. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Enterprise version of vmalert supports S3 and GCS paths to rules. @@ -1179,6 +1180,7 @@ The shortlist of configuration flags is the following: -rule.templates="/path/to/file". Path to a single file with go templates -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, absolute path to all .tpl files in root. + -rule.templates="dir/**/*.tpl". Includes to all .tpl files in "dir" folder and it's subfolders recursively. Supports an array of values separated by comma or specified via multiple flags. -rule.updateEntriesLimit int Defines the max number of rule's state updates stored in-memory. Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param. (default 20) diff --git a/app/vmalert/config/fslocal/fslocal.go b/app/vmalert/config/fslocal/fslocal.go index 6ebfaac17..a3a1bb715 100644 --- a/app/vmalert/config/fslocal/fslocal.go +++ b/app/vmalert/config/fslocal/fslocal.go @@ -3,7 +3,8 @@ package fslocal import ( "fmt" "os" - "path/filepath" + + "github.com/bmatcuk/doublestar/v4" ) // FS represents a local file system @@ -16,7 +17,7 @@ type FS struct { // Init verifies that configured Pattern is correct func (fs *FS) Init() error { - _, err := filepath.Glob(fs.Pattern) + _, err := doublestar.FilepathGlob(fs.Pattern) return err } @@ -27,7 +28,7 @@ func (fs *FS) String() string { // List returns the list of file names which will be read via Read fn func (fs *FS) List() ([]string, error) { - matches, err := filepath.Glob(fs.Pattern) + matches, err := doublestar.FilepathGlob(fs.Pattern) if err != nil { return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err) } diff --git a/app/vmalert/main.go b/app/vmalert/main.go index fa40ea6c9..8b49029e2 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/VictoriaMetrics/metrics" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" @@ -24,7 +26,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics" - "github.com/VictoriaMetrics/metrics" ) var ( @@ -33,6 +34,7 @@ Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/**/*.yaml". Includes to all .yaml files in "dir" folder and it's subfolders recursively. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Enterprise version of vmalert supports S3 and GCS paths to rules. @@ -47,7 +49,9 @@ See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-stor Examples: -rule.templates="/path/to/file". Path to a single file with go templates -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, -absolute path to all .tpl files in root.`) +absolute path to all .tpl files in root. + -rule.templates="dir/**/*.tpl". Includes to all .tpl files in "dir" folder and it's subfolders recursively. +`) rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+ "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead") diff --git a/app/vmalert/templates/template.go b/app/vmalert/templates/template.go index fb4d6ecfe..08d81b00e 100644 --- a/app/vmalert/templates/template.go +++ b/app/vmalert/templates/template.go @@ -21,7 +21,6 @@ import ( "math" "net" "net/url" - "path/filepath" "regexp" "sort" "strconv" @@ -30,6 +29,8 @@ import ( textTpl "text/template" "time" + "github.com/bmatcuk/doublestar/v4" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" @@ -59,12 +60,12 @@ func Load(pathPatterns []string, overwrite bool) error { var err error tmpl := newTemplate() for _, tp := range pathPatterns { - p, err := filepath.Glob(tp) + p, err := doublestar.FilepathGlob(tp) if err != nil { return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err) } if len(p) > 0 { - tmpl, err = tmpl.ParseGlob(tp) + tmpl, err = tmpl.ParseFiles(p...) if err != nil { return fmt.Errorf("failed to parse template glob %q: %w", tp, err) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e480a72de..3edf8882d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): integrate WITH template playground. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3811). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add a comparison of data from the previous day with data from the current day to the `Cardinality Explorer`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3967). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): display histogram metrics as a heatmap in the `explore metrics` tab. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4111). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support of recursive globs for `-rule` and `-rule.templates` by using `**` in the glob pattern. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4041). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the metric relabel playground feature to the vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3807). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084). diff --git a/docs/vmalert.md b/docs/vmalert.md index e2589e218..abf2232eb 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1160,7 +1160,8 @@ The shortlist of configuration flags is the following: Supports hierarchical patterns and regexpes. Examples: -rule="/path/to/file". Path to a single file with alerting rules - -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/**/*.yaml". Includes to all .yaml files in "dir" folder and it's subfolders recursively. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Enterprise version of vmalert supports S3 and GCS paths to rules. @@ -1183,6 +1184,7 @@ The shortlist of configuration flags is the following: -rule.templates="/path/to/file". Path to a single file with go templates -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder, absolute path to all .tpl files in root. + -rule.templates="dir/**/*.tpl". Includes to all .tpl files in "dir" folder and it's subfolders recursively. Supports an array of values separated by comma or specified via multiple flags. -rule.updateEntriesLimit int Defines the max number of rule's state updates stored in-memory. Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param. (default 20) diff --git a/go.mod b/go.mod index 8bff52b66..261dbee43 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,8 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) +require github.com/bmatcuk/doublestar/v4 v4.6.0 + require ( cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.19.0 // indirect diff --git a/go.sum b/go.sum index 156e011da..9acc69c19 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= +github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= diff --git a/vendor/github.com/bmatcuk/doublestar/v4/.codecov.yml b/vendor/github.com/bmatcuk/doublestar/v4/.codecov.yml new file mode 100644 index 000000000..db6e504a9 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/.codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + threshold: 1% + patch: + default: + target: 70% +ignore: + - globoptions.go diff --git a/vendor/github.com/bmatcuk/doublestar/v4/.gitignore b/vendor/github.com/bmatcuk/doublestar/v4/.gitignore new file mode 100644 index 000000000..af212ecc2 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/.gitignore @@ -0,0 +1,32 @@ +# vi +*~ +*.swp +*.swo + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# test directory +test/ diff --git a/vendor/github.com/bmatcuk/doublestar/v4/LICENSE b/vendor/github.com/bmatcuk/doublestar/v4/LICENSE new file mode 100644 index 000000000..309c9d1d1 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Bob Matcuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/bmatcuk/doublestar/v4/README.md b/vendor/github.com/bmatcuk/doublestar/v4/README.md new file mode 100644 index 000000000..be715ff69 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/README.md @@ -0,0 +1,404 @@ +# doublestar + +Path pattern matching and globbing supporting `doublestar` (`**`) patterns. + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/bmatcuk/doublestar)](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4) +[![Release](https://img.shields.io/github/release/bmatcuk/doublestar.svg?branch=master)](https://github.com/bmatcuk/doublestar/releases) +[![Build Status](https://github.com/bmatcuk/doublestar/actions/workflows/test.yml/badge.svg)](https://github.com/bmatcuk/doublestar/actions) +[![codecov.io](https://img.shields.io/codecov/c/github/bmatcuk/doublestar.svg?branch=master)](https://codecov.io/github/bmatcuk/doublestar?branch=master) +[![Sponsor](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/bmatcuk) + +## About + +#### [Upgrading?](UPGRADING.md) + +**doublestar** is a [golang] implementation of path pattern matching and +globbing with support for "doublestar" (aka globstar: `**`) patterns. + +doublestar patterns match files and directories recursively. For example, if +you had the following directory structure: + +```bash +grandparent +`-- parent + |-- child1 + `-- child2 +``` + +You could find the children with patterns such as: `**/child*`, +`grandparent/**/child?`, `**/parent/*`, or even just `**` by itself (which will +return all files and directories recursively). + +Bash's globstar is doublestar's inspiration and, as such, works similarly. +Note that the doublestar must appear as a path component by itself. A pattern +such as `/path**` is invalid and will be treated the same as `/path*`, but +`/path*/**` should achieve the desired result. Additionally, `/path/**` will +match all directories and files under the path directory, but `/path/**/` will +only match directories. + +v4 is a complete rewrite with a focus on performance. Additionally, +[doublestar] has been updated to use the new [io/fs] package for filesystem +access. As a result, it is only supported by [golang] v1.16+. + +## Installation + +**doublestar** can be installed via `go get`: + +```bash +go get github.com/bmatcuk/doublestar/v4 +``` + +To use it in your code, you must import it: + +```go +import "github.com/bmatcuk/doublestar/v4" +``` + +## Usage + +### ErrBadPattern + +```go +doublestar.ErrBadPattern +``` + +Returned by various functions to report that the pattern is malformed. At the +moment, this value is equal to `path.ErrBadPattern`, but, for portability, this +equivalence should probably not be relied upon. + +### Match + +```go +func Match(pattern, name string) (bool, error) +``` + +Match returns true if `name` matches the file name `pattern` ([see +"patterns"]). `name` and `pattern` are split on forward slash (`/`) characters +and may be relative or absolute. + +Match requires pattern to match all of name, not just a substring. The only +possible returned error is `ErrBadPattern`, when pattern is malformed. + +Note: this is meant as a drop-in replacement for `path.Match()` which always +uses `'/'` as the path separator. If you want to support systems which use a +different path separator (such as Windows), what you want is `PathMatch()`. +Alternatively, you can run `filepath.ToSlash()` on both pattern and name and +then use this function. + +Note: users should _not_ count on the returned error, +`doublestar.ErrBadPattern`, being equal to `path.ErrBadPattern`. + + +### PathMatch + +```go +func PathMatch(pattern, name string) (bool, error) +``` + +PathMatch returns true if `name` matches the file name `pattern` ([see +"patterns"]). The difference between Match and PathMatch is that PathMatch will +automatically use your system's path separator to split `name` and `pattern`. +On systems where the path separator is `'\'`, escaping will be disabled. + +Note: this is meant as a drop-in replacement for `filepath.Match()`. It assumes +that both `pattern` and `name` are using the system's path separator. If you +can't be sure of that, use `filepath.ToSlash()` on both `pattern` and `name`, +and then use the `Match()` function instead. + +### GlobOption + +Options that may be passed to `Glob`, `GlobWalk`, or `FilepathGlob`. Any number +of options may be passed to these functions, and in any order, as the last +argument(s). + +```go +WithFailOnIOErrors() +``` + +If passed, doublestar will abort and return IO errors when encountered. Note +that if the glob pattern references a path that does not exist (such as +`nonexistent/path/*`), this is _not_ considered an IO error: it is considered a +pattern with no matches. + +```go +WithFailOnPatternNotExist() +``` + +If passed, doublestar will abort and return `doublestar.ErrPatternNotExist` if +the pattern references a path that does not exist before any meta characters +such as `nonexistent/path/*`. Note that alts (ie, `{...}`) are expanded before +this check. In other words, a pattern such as `{a,b}/*` may fail if either `a` +or `b` do not exist but `*/{a,b}` will never fail because the star may match +nothing. + +```go +WithFilesOnly() +``` + +If passed, doublestar will only return "files" from `Glob`, `GlobWalk`, or +`FilepathGlob`. In this context, "files" are anything that is not a directory +or a symlink to a directory. + +Note: if combined with the WithNoFollow option, symlinks to directories _will_ +be included in the result since no attempt is made to follow the symlink. + +```go +WithNoFollow() +``` + +If passed, doublestar will not follow symlinks while traversing the filesystem. +However, due to io/fs's _very_ poor support for querying the filesystem about +symlinks, there's a caveat here: if part of the pattern before any meta +characters contains a reference to a symlink, it will be followed. For example, +a pattern such as `path/to/symlink/*` will be followed assuming it is a valid +symlink to a directory. However, from this same example, a pattern such as +`path/to/**` will not traverse the `symlink`, nor would `path/*/symlink/*` + +Note: if combined with the WithFilesOnly option, symlinks to directories _will_ +be included in the result since no attempt is made to follow the symlink. + +### Glob + +```go +func Glob(fsys fs.FS, pattern string, opts ...GlobOption) ([]string, error) +``` + +Glob returns the names of all files matching pattern or nil if there is no +matching file. The syntax of patterns is the same as in `Match()`. The pattern +may describe hierarchical names such as `usr/*/bin/ed`. + +Glob ignores file system errors such as I/O errors reading directories by +default. The only possible returned error is `ErrBadPattern`, reporting that +the pattern is malformed. + +To enable aborting on I/O errors, the `WithFailOnIOErrors` option can be +passed. + +Note: this is meant as a drop-in replacement for `io/fs.Glob()`. Like +`io/fs.Glob()`, this function assumes that your pattern uses `/` as the path +separator even if that's not correct for your OS (like Windows). If you aren't +sure if that's the case, you can use `filepath.ToSlash()` on your pattern +before calling `Glob()`. + +Like `io/fs.Glob()`, patterns containing `/./`, `/../`, or starting with `/` +will return no results and no errors. This seems to be a [conscious +decision](https://github.com/golang/go/issues/44092#issuecomment-774132549), +even if counter-intuitive. You can use [SplitPattern] to divide a pattern into +a base path (to initialize an `FS` object) and pattern. + +Note: users should _not_ count on the returned error, +`doublestar.ErrBadPattern`, being equal to `path.ErrBadPattern`. + +### GlobWalk + +```go +type GlobWalkFunc func(path string, d fs.DirEntry) error + +func GlobWalk(fsys fs.FS, pattern string, fn GlobWalkFunc, opts ...GlobOption) error +``` + +GlobWalk calls the callback function `fn` for every file matching pattern. The +syntax of pattern is the same as in Match() and the behavior is the same as +Glob(), with regard to limitations (such as patterns containing `/./`, `/../`, +or starting with `/`). The pattern may describe hierarchical names such as +usr/*/bin/ed. + +GlobWalk may have a small performance benefit over Glob if you do not need a +slice of matches because it can avoid allocating memory for the matches. +Additionally, GlobWalk gives you access to the `fs.DirEntry` objects for each +match, and lets you quit early by returning a non-nil error from your callback +function. Like `io/fs.WalkDir`, if your callback returns `SkipDir`, GlobWalk +will skip the current directory. This means that if the current path _is_ a +directory, GlobWalk will not recurse into it. If the current path is not a +directory, the rest of the parent directory will be skipped. + +GlobWalk ignores file system errors such as I/O errors reading directories by +default. GlobWalk may return `ErrBadPattern`, reporting that the pattern is +malformed. + +To enable aborting on I/O errors, the `WithFailOnIOErrors` option can be +passed. + +Additionally, if the callback function `fn` returns an error, GlobWalk will +exit immediately and return that error. + +Like Glob(), this function assumes that your pattern uses `/` as the path +separator even if that's not correct for your OS (like Windows). If you aren't +sure if that's the case, you can use filepath.ToSlash() on your pattern before +calling GlobWalk(). + +Note: users should _not_ count on the returned error, +`doublestar.ErrBadPattern`, being equal to `path.ErrBadPattern`. + +### FilepathGlob + +```go +func FilepathGlob(pattern string, opts ...GlobOption) (matches []string, err error) +``` + +FilepathGlob returns the names of all files matching pattern or nil if there is +no matching file. The syntax of pattern is the same as in Match(). The pattern +may describe hierarchical names such as usr/*/bin/ed. + +FilepathGlob ignores file system errors such as I/O errors reading directories +by default. The only possible returned error is `ErrBadPattern`, reporting that +the pattern is malformed. + +To enable aborting on I/O errors, the `WithFailOnIOErrors` option can be +passed. + +Note: FilepathGlob is a convenience function that is meant as a drop-in +replacement for `path/filepath.Glob()` for users who don't need the +complication of io/fs. Basically, it: + +* Runs `filepath.Clean()` and `ToSlash()` on the pattern +* Runs `SplitPattern()` to get a base path and a pattern to Glob +* Creates an FS object from the base path and `Glob()s` on the pattern +* Joins the base path with all of the matches from `Glob()` + +Returned paths will use the system's path separator, just like +`filepath.Glob()`. + +Note: the returned error `doublestar.ErrBadPattern` is not equal to +`filepath.ErrBadPattern`. + +### SplitPattern + +```go +func SplitPattern(p string) (base, pattern string) +``` + +SplitPattern is a utility function. Given a pattern, SplitPattern will return +two strings: the first string is everything up to the last slash (`/`) that +appears _before_ any unescaped "meta" characters (ie, `*?[{`). The second +string is everything after that slash. For example, given the pattern: + +``` +../../path/to/meta*/** + ^----------- split here +``` + +SplitPattern returns "../../path/to" and "meta*/**". This is useful for +initializing os.DirFS() to call Glob() because Glob() will silently fail if +your pattern includes `/./` or `/../`. For example: + +```go +base, pattern := SplitPattern("../../path/to/meta*/**") +fsys := os.DirFS(base) +matches, err := Glob(fsys, pattern) +``` + +If SplitPattern cannot find somewhere to split the pattern (for example, +`meta*/**`), it will return "." and the unaltered pattern (`meta*/**` in this +example). + +Of course, it is your responsibility to decide if the returned base path is +"safe" in the context of your application. Perhaps you could use Match() to +validate against a list of approved base directories? + +### ValidatePattern + +```go +func ValidatePattern(s string) bool +``` + +Validate a pattern. Patterns are validated while they run in Match(), +PathMatch(), and Glob(), so, you normally wouldn't need to call this. However, +there are cases where this might be useful: for example, if your program allows +a user to enter a pattern that you'll run at a later time, you might want to +validate it. + +ValidatePattern assumes your pattern uses '/' as the path separator. + +### ValidatePathPattern + +```go +func ValidatePathPattern(s string) bool +``` + +Like ValidatePattern, only uses your OS path separator. In other words, use +ValidatePattern if you would normally use Match() or Glob(). Use +ValidatePathPattern if you would normally use PathMatch(). Keep in mind, Glob() +requires '/' separators, even if your OS uses something else. + +### Patterns + +**doublestar** supports the following special terms in the patterns: + +Special Terms | Meaning +------------- | ------- +`*` | matches any sequence of non-path-separators +`/**/` | matches zero or more directories +`?` | matches any single non-path-separator character +`[class]` | matches any single non-path-separator character against a class of characters ([see "character classes"]) +`{alt1,...}` | matches a sequence of characters if one of the comma-separated alternatives matches + +Any character with a special meaning can be escaped with a backslash (`\`). + +A doublestar (`**`) should appear surrounded by path separators such as `/**/`. +A mid-pattern doublestar (`**`) behaves like bash's globstar option: a pattern +such as `path/to/**.txt` would return the same results as `path/to/*.txt`. The +pattern you're looking for is `path/to/**/*.txt`. + +#### Character Classes + +Character classes support the following: + +Class | Meaning +---------- | ------- +`[abc]` | matches any single character within the set +`[a-z]` | matches any single character in the range +`[^class]` | matches any single character which does *not* match the class +`[!class]` | same as `^`: negates the class + +## Performance + +``` +goos: darwin +goarch: amd64 +pkg: github.com/bmatcuk/doublestar/v4 +cpu: Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz +BenchmarkMatch-8 285639 3868 ns/op 0 B/op 0 allocs/op +BenchmarkGoMatch-8 286945 3726 ns/op 0 B/op 0 allocs/op +BenchmarkPathMatch-8 320511 3493 ns/op 0 B/op 0 allocs/op +BenchmarkGoPathMatch-8 304236 3434 ns/op 0 B/op 0 allocs/op +BenchmarkGlob-8 466 2501123 ns/op 190225 B/op 2849 allocs/op +BenchmarkGlobWalk-8 476 2536293 ns/op 184017 B/op 2750 allocs/op +BenchmarkGoGlob-8 463 2574836 ns/op 194249 B/op 2929 allocs/op +``` + +These benchmarks (in `doublestar_test.go`) compare Match() to path.Match(), +PathMath() to filepath.Match(), and Glob() + GlobWalk() to io/fs.Glob(). They +only run patterns that the standard go packages can understand as well (so, no +`{alts}` or `**`) for a fair comparison. Of course, alts and doublestars will +be less performant than the other pattern meta characters. + +Alts are essentially like running multiple patterns, the number of which can +get large if your pattern has alts nested inside alts. This affects both +matching (ie, Match()) and globbing (Glob()). + +`**` performance in matching is actually pretty similar to a regular `*`, but +can cause a large number of reads when globbing as it will need to recursively +traverse your filesystem. + +## Sponsors +I started this project in 2014 in my spare time and have been maintaining it +ever since. In that time, it has grown into one of the most popular globbing +libraries in the Go ecosystem. So, if **doublestar** is a useful library in +your project, consider [sponsoring] my work! I'd really appreciate it! + +[![reviewpad](../sponsors/reviewpad.png?raw=true)](https://reviewpad.com/) + +Thanks for sponsoring me! + +## License + +[MIT License](LICENSE) + +[SplitPattern]: #splitpattern +[doublestar]: https://github.com/bmatcuk/doublestar +[golang]: http://golang.org/ +[io/fs]: https://pkg.go.dev/io/fs +[see "character classes"]: #character-classes +[see "patterns"]: #patterns +[sponsoring]: https://github.com/sponsors/bmatcuk diff --git a/vendor/github.com/bmatcuk/doublestar/v4/UPGRADING.md b/vendor/github.com/bmatcuk/doublestar/v4/UPGRADING.md new file mode 100644 index 000000000..25aace3db --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/UPGRADING.md @@ -0,0 +1,63 @@ +# Upgrading from v3 to v4 + +v4 is a complete rewrite with a focus on performance. Additionally, +[doublestar] has been updated to use the new [io/fs] package for filesystem +access. As a result, it is only supported by [golang] v1.16+. + +`Match()` and `PathMatch()` mostly did not change, besides big performance +improvements. Their API is the same. However, note the following corner cases: + +* In previous versions of [doublestar], `PathMatch()` could accept patterns + that used either platform-specific path separators, or `/`. This was + undocumented and didn't match `filepath.Match()`. In v4, both `pattern` and + `name` must be using appropriate path separators for the platform. You can + use `filepath.FromSlash()` to change `/` to platform-specific separators if + you aren't sure. +* In previous versions of [doublestar], a pattern such as `path/to/a/**` would + _not_ match `path/to/a`. In v4, this pattern _will_ match because if `a` was + a directory, `Glob()` would return it. In other words, the following returns + true: `Match("path/to/a/**", "path/to/a")` + +`Glob()` changed from using a [doublestar]-specific filesystem abstraction (the +`OS` interface) to the [io/fs] package. As a result, it now takes a `fs.FS` as +its first argument. This change has a couple ramifications: + +* Like `io/fs.Glob`, `pattern` must use a `/` as path separator, even on + platforms that use something else. You can use `filepath.ToSlash()` on your + patterns if you aren't sure. +* Patterns that contain `/./` or `/../` are invalid. The [io/fs] package + rejects them, returning an IO error. Since `Glob()` ignores IO errors, it'll + end up being silently rejected. You can run `path.Clean()` to ensure they are + removed from the pattern. + +v4 also added a `GlobWalk()` function that is slightly more performant than +`Glob()` if you just need to iterate over the results and don't need a string +slice. You also get `fs.DirEntry` objects for each result, and can quit early +if your callback returns an error. + +# Upgrading from v2 to v3 + +v3 introduced using `!` to negate character classes, in addition to `^`. If any +of your patterns include a character class that starts with an exclamation mark +(ie, `[!...]`), you'll need to update the pattern to escape or move the +exclamation mark. Note that, like the caret (`^`), it only negates the +character class if it is the first character in the character class. + +# Upgrading from v1 to v2 + +The change from v1 to v2 was fairly minor: the return type of the `Open` method +on the `OS` interface was changed from `*os.File` to `File`, a new interface +exported by doublestar. The new `File` interface only defines the functionality +doublestar actually needs (`io.Closer` and `Readdir`), making it easier to use +doublestar with [go-billy], [afero], or something similar. If you were using +this functionality, updating should be as easy as updating `Open's` return +type, since `os.File` already implements `doublestar.File`. + +If you weren't using this functionality, updating should be as easy as changing +your dependencies to point to v2. + +[afero]: https://github.com/spf13/afero +[doublestar]: https://github.com/bmatcuk/doublestar +[go-billy]: https://github.com/src-d/go-billy +[golang]: http://golang.org/ +[io/fs]: https://golang.org/pkg/io/fs/ diff --git a/vendor/github.com/bmatcuk/doublestar/v4/doublestar.go b/vendor/github.com/bmatcuk/doublestar/v4/doublestar.go new file mode 100644 index 000000000..210fd40ce --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/doublestar.go @@ -0,0 +1,13 @@ +package doublestar + +import ( + "errors" + "path" +) + +// ErrBadPattern indicates a pattern was malformed. +var ErrBadPattern = path.ErrBadPattern + +// ErrPatternNotExist indicates that the pattern passed to Glob, GlobWalk, or +// FilepathGlob references a path that does not exist. +var ErrPatternNotExist = errors.New("pattern does not exist") diff --git a/vendor/github.com/bmatcuk/doublestar/v4/glob.go b/vendor/github.com/bmatcuk/doublestar/v4/glob.go new file mode 100644 index 000000000..519601b15 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/glob.go @@ -0,0 +1,473 @@ +package doublestar + +import ( + "errors" + "io/fs" + "path" +) + +// Glob returns the names of all files matching pattern or nil if there is no +// matching file. The syntax of pattern is the same as in Match(). The pattern +// may describe hierarchical names such as usr/*/bin/ed. +// +// Glob ignores file system errors such as I/O errors reading directories by +// default. The only possible returned error is ErrBadPattern, reporting that +// the pattern is malformed. +// +// To enable aborting on I/O errors, the WithFailOnIOErrors option can be +// passed. +// +// Note: this is meant as a drop-in replacement for io/fs.Glob(). Like +// io/fs.Glob(), this function assumes that your pattern uses `/` as the path +// separator even if that's not correct for your OS (like Windows). If you +// aren't sure if that's the case, you can use filepath.ToSlash() on your +// pattern before calling Glob(). +// +// Like `io/fs.Glob()`, patterns containing `/./`, `/../`, or starting with `/` +// will return no results and no errors. You can use SplitPattern to divide a +// pattern into a base path (to initialize an `FS` object) and pattern. +// +// Note: users should _not_ count on the returned error, +// doublestar.ErrBadPattern, being equal to path.ErrBadPattern. +// +func Glob(fsys fs.FS, pattern string, opts ...GlobOption) ([]string, error) { + if !ValidatePattern(pattern) { + return nil, ErrBadPattern + } + + g := newGlob(opts...) + + if hasMidDoubleStar(pattern) { + // If the pattern has a `**` anywhere but the very end, GlobWalk is more + // performant because it can get away with less allocations. If the pattern + // ends in a `**`, both methods are pretty much the same, but Glob has a + // _very_ slight advantage because of lower function call overhead. + var matches []string + err := g.doGlobWalk(fsys, pattern, true, true, func(p string, d fs.DirEntry) error { + matches = append(matches, p) + return nil + }) + return matches, err + } + return g.doGlob(fsys, pattern, nil, true, true) +} + +// Does the actual globbin' +// - firstSegment is true if we're in the first segment of the pattern, ie, +// the right-most part where we can match files. If it's false, we're +// somewhere in the middle (or at the beginning) and can only match +// directories since there are path segments above us. +// - beforeMeta is true if we're exploring segments before any meta +// characters, ie, in a pattern such as `path/to/file*.txt`, the `path/to/` +// bit does not contain any meta characters. +func (g *glob) doGlob(fsys fs.FS, pattern string, m []string, firstSegment, beforeMeta bool) (matches []string, err error) { + matches = m + patternStart := indexMeta(pattern) + if patternStart == -1 { + // pattern doesn't contain any meta characters - does a file matching the + // pattern exist? + // The pattern may contain escaped wildcard characters for an exact path match. + path := unescapeMeta(pattern) + pathInfo, pathExists, pathErr := g.exists(fsys, path, beforeMeta) + if pathErr != nil { + return nil, pathErr + } + + if pathExists && (!firstSegment || !g.filesOnly || !pathInfo.IsDir()) { + matches = append(matches, path) + } + + return + } + + dir := "." + splitIdx := lastIndexSlashOrAlt(pattern) + if splitIdx != -1 { + if pattern[splitIdx] == '}' { + openingIdx := indexMatchedOpeningAlt(pattern[:splitIdx]) + if openingIdx == -1 { + // if there's no matching opening index, technically Match() will treat + // an unmatched `}` as nothing special, so... we will, too! + splitIdx = lastIndexSlash(pattern[:splitIdx]) + if splitIdx != -1 { + dir = pattern[:splitIdx] + pattern = pattern[splitIdx+1:] + } + } else { + // otherwise, we have to handle the alts: + return g.globAlts(fsys, pattern, openingIdx, splitIdx, matches, firstSegment, beforeMeta) + } + } else { + dir = pattern[:splitIdx] + pattern = pattern[splitIdx+1:] + } + } + + // if `splitIdx` is less than `patternStart`, we know `dir` has no meta + // characters. They would be equal if they are both -1, which means `dir` + // will be ".", and we know that doesn't have meta characters either. + if splitIdx <= patternStart { + return g.globDir(fsys, dir, pattern, matches, firstSegment, beforeMeta) + } + + var dirs []string + dirs, err = g.doGlob(fsys, dir, matches, false, beforeMeta) + if err != nil { + return + } + for _, d := range dirs { + matches, err = g.globDir(fsys, d, pattern, matches, firstSegment, false) + if err != nil { + return + } + } + + return +} + +// handle alts in the glob pattern - `openingIdx` and `closingIdx` are the +// indexes of `{` and `}`, respectively +func (g *glob) globAlts(fsys fs.FS, pattern string, openingIdx, closingIdx int, m []string, firstSegment, beforeMeta bool) (matches []string, err error) { + matches = m + + var dirs []string + startIdx := 0 + afterIdx := closingIdx + 1 + splitIdx := lastIndexSlashOrAlt(pattern[:openingIdx]) + if splitIdx == -1 || pattern[splitIdx] == '}' { + // no common prefix + dirs = []string{""} + } else { + // our alts have a common prefix that we can process first + dirs, err = g.doGlob(fsys, pattern[:splitIdx], matches, false, beforeMeta) + if err != nil { + return + } + + startIdx = splitIdx + 1 + } + + for _, d := range dirs { + patIdx := openingIdx + 1 + altResultsStartIdx := len(matches) + thisResultStartIdx := altResultsStartIdx + for patIdx < closingIdx { + nextIdx := indexNextAlt(pattern[patIdx:closingIdx], true) + if nextIdx == -1 { + nextIdx = closingIdx + } else { + nextIdx += patIdx + } + + alt := buildAlt(d, pattern, startIdx, openingIdx, patIdx, nextIdx, afterIdx) + matches, err = g.doGlob(fsys, alt, matches, firstSegment, beforeMeta) + if err != nil { + return + } + + matchesLen := len(matches) + if altResultsStartIdx != thisResultStartIdx && thisResultStartIdx != matchesLen { + // Alts can result in matches that aren't sorted, or, worse, duplicates + // (consider the trivial pattern `path/to/{a,*}`). Since doGlob returns + // sorted results, we can do a sort of in-place merge and remove + // duplicates. But, we only need to do this if this isn't the first alt + // (ie, `altResultsStartIdx != thisResultsStartIdx`) and if the latest + // alt actually added some matches (`thisResultStartIdx != + // len(matches)`) + matches = sortAndRemoveDups(matches, altResultsStartIdx, thisResultStartIdx, matchesLen) + + // length of matches may have changed + thisResultStartIdx = len(matches) + } else { + thisResultStartIdx = matchesLen + } + + patIdx = nextIdx + 1 + } + } + + return +} + +// find files/subdirectories in the given `dir` that match `pattern` +func (g *glob) globDir(fsys fs.FS, dir, pattern string, matches []string, canMatchFiles, beforeMeta bool) (m []string, e error) { + m = matches + + if pattern == "" { + if !canMatchFiles || !g.filesOnly { + // pattern can be an empty string if the original pattern ended in a + // slash, in which case, we should just return dir, but only if it + // actually exists and it's a directory (or a symlink to a directory) + _, isDir, err := g.isPathDir(fsys, dir, beforeMeta) + if err != nil { + return nil, err + } + if isDir { + m = append(m, dir) + } + } + return + } + + if pattern == "**" { + return g.globDoubleStar(fsys, dir, m, canMatchFiles, beforeMeta) + } + + dirs, err := fs.ReadDir(fsys, dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + e = g.handlePatternNotExist(beforeMeta) + } else { + e = g.forwardErrIfFailOnIOErrors(err) + } + return + } + + var matched bool + for _, info := range dirs { + name := info.Name() + matched, e = matchWithSeparator(pattern, name, '/', false) + if e != nil { + return + } + if matched { + matched = canMatchFiles + if !matched || g.filesOnly { + matched, e = g.isDir(fsys, dir, name, info) + if e != nil { + return + } + if canMatchFiles { + // if we're here, it's because g.filesOnly + // is set and we don't want directories + matched = !matched + } + } + if matched { + m = append(m, path.Join(dir, name)) + } + } + } + + return +} + +func (g *glob) globDoubleStar(fsys fs.FS, dir string, matches []string, canMatchFiles, beforeMeta bool) ([]string, error) { + dirs, err := fs.ReadDir(fsys, dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return matches, g.handlePatternNotExist(beforeMeta) + } else { + return matches, g.forwardErrIfFailOnIOErrors(err) + } + } + + if !g.filesOnly { + // `**` can match *this* dir, so add it + matches = append(matches, dir) + } + + for _, info := range dirs { + name := info.Name() + isDir, err := g.isDir(fsys, dir, name, info) + if err != nil { + return nil, err + } + if isDir { + matches, err = g.globDoubleStar(fsys, path.Join(dir, name), matches, canMatchFiles, false) + if err != nil { + return nil, err + } + } else if canMatchFiles { + matches = append(matches, path.Join(dir, name)) + } + } + + return matches, nil +} + +// Returns true if the pattern has a doublestar in the middle of the pattern. +// In this case, GlobWalk is faster because it can get away with less +// allocations. However, Glob has a _very_ slight edge if the pattern ends in +// `**`. +func hasMidDoubleStar(p string) bool { + // subtract 3: 2 because we want to return false if the pattern ends in `**` + // (Glob is _very_ slightly faster in that case), and the extra 1 because our + // loop checks p[i] and p[i+1]. + l := len(p) - 3 + for i := 0; i < l; i++ { + if p[i] == '\\' { + // escape next byte + i++ + } else if p[i] == '*' && p[i+1] == '*' { + return true + } + } + return false +} + +// Returns the index of the first unescaped meta character, or negative 1. +func indexMeta(s string) int { + var c byte + l := len(s) + for i := 0; i < l; i++ { + c = s[i] + if c == '*' || c == '?' || c == '[' || c == '{' { + return i + } else if c == '\\' { + // skip next byte + i++ + } + } + return -1 +} + +// Returns the index of the last unescaped slash or closing alt (`}`) in the +// string, or negative 1. +func lastIndexSlashOrAlt(s string) int { + for i := len(s) - 1; i >= 0; i-- { + if (s[i] == '/' || s[i] == '}') && (i == 0 || s[i-1] != '\\') { + return i + } + } + return -1 +} + +// Returns the index of the last unescaped slash in the string, or negative 1. +func lastIndexSlash(s string) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '/' && (i == 0 || s[i-1] != '\\') { + return i + } + } + return -1 +} + +// Assuming the byte after the end of `s` is a closing `}`, this function will +// find the index of the matching `{`. That is, it'll skip over any nested `{}` +// and account for escaping. +func indexMatchedOpeningAlt(s string) int { + alts := 1 + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '}' && (i == 0 || s[i-1] != '\\') { + alts++ + } else if s[i] == '{' && (i == 0 || s[i-1] != '\\') { + if alts--; alts == 0 { + return i + } + } + } + return -1 +} + +// Returns true if the path exists +func (g *glob) exists(fsys fs.FS, name string, beforeMeta bool) (fs.FileInfo, bool, error) { + // name might end in a slash, but Stat doesn't like that + namelen := len(name) + if namelen > 1 && name[namelen-1] == '/' { + name = name[:namelen-1] + } + + info, err := fs.Stat(fsys, name) + if errors.Is(err, fs.ErrNotExist) { + return nil, false, g.handlePatternNotExist(beforeMeta) + } + return info, err == nil, g.forwardErrIfFailOnIOErrors(err) +} + +// Returns true if the path exists and is a directory or a symlink to a +// directory +func (g *glob) isPathDir(fsys fs.FS, name string, beforeMeta bool) (fs.FileInfo, bool, error) { + info, err := fs.Stat(fsys, name) + if errors.Is(err, fs.ErrNotExist) { + return nil, false, g.handlePatternNotExist(beforeMeta) + } + return info, err == nil && info.IsDir(), g.forwardErrIfFailOnIOErrors(err) +} + +// Returns whether or not the given DirEntry is a directory. If the DirEntry +// represents a symbolic link, the link is followed by running fs.Stat() on +// `path.Join(dir, name)` (if dir is "", name will be used without joining) +func (g *glob) isDir(fsys fs.FS, dir, name string, info fs.DirEntry) (bool, error) { + if !g.noFollow && (info.Type()&fs.ModeSymlink) > 0 { + p := name + if dir != "" { + p = path.Join(dir, name) + } + finfo, err := fs.Stat(fsys, p) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // this function is only ever called while expanding a glob, so it can + // never return ErrPatternNotExist + return false, nil + } + return false, g.forwardErrIfFailOnIOErrors(err) + } + return finfo.IsDir(), nil + } + return info.IsDir(), nil +} + +// Builds a string from an alt +func buildAlt(prefix, pattern string, startIdx, openingIdx, currentIdx, nextIdx, afterIdx int) string { + // pattern: + // ignored/start{alts,go,here}remaining - len = 36 + // | | | | ^--- afterIdx = 27 + // | | | \--------- nextIdx = 21 + // | | \----------- currentIdx = 19 + // | \----------------- openingIdx = 13 + // \---------------------- startIdx = 8 + // + // result: + // prefix/startgoremaining - len = 7 + 5 + 2 + 9 = 23 + var buf []byte + patLen := len(pattern) + size := (openingIdx - startIdx) + (nextIdx - currentIdx) + (patLen - afterIdx) + if prefix != "" && prefix != "." { + buf = make([]byte, 0, size+len(prefix)+1) + buf = append(buf, prefix...) + buf = append(buf, '/') + } else { + buf = make([]byte, 0, size) + } + buf = append(buf, pattern[startIdx:openingIdx]...) + buf = append(buf, pattern[currentIdx:nextIdx]...) + if afterIdx < patLen { + buf = append(buf, pattern[afterIdx:]...) + } + return string(buf) +} + +// Running alts can produce results that are not sorted, and, worse, can cause +// duplicates (consider the trivial pattern `path/to/{a,*}`). Since we know +// each run of doGlob is sorted, we can basically do the "merge" step of a +// merge sort in-place. +func sortAndRemoveDups(matches []string, idx1, idx2, l int) []string { + var tmp string + for ; idx1 < idx2; idx1++ { + if matches[idx1] < matches[idx2] { + // order is correct + continue + } else if matches[idx1] > matches[idx2] { + // need to swap and then re-sort matches above idx2 + tmp = matches[idx1] + matches[idx1] = matches[idx2] + + shft := idx2 + 1 + for ; shft < l && matches[shft] < tmp; shft++ { + matches[shft-1] = matches[shft] + } + matches[shft-1] = tmp + } else { + // duplicate - shift matches above idx2 down one and decrement l + for shft := idx2 + 1; shft < l; shft++ { + matches[shft-1] = matches[shft] + } + if l--; idx2 == l { + // nothing left to do... matches[idx2:] must have been full of dups + break + } + } + } + return matches[:l] +} diff --git a/vendor/github.com/bmatcuk/doublestar/v4/globoptions.go b/vendor/github.com/bmatcuk/doublestar/v4/globoptions.go new file mode 100644 index 000000000..9483c4bb0 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/globoptions.go @@ -0,0 +1,144 @@ +package doublestar + +import "strings" + +// glob is an internal type to store options during globbing. +type glob struct { + failOnIOErrors bool + failOnPatternNotExist bool + filesOnly bool + noFollow bool +} + +// GlobOption represents a setting that can be passed to Glob, GlobWalk, and +// FilepathGlob. +type GlobOption func(*glob) + +// Construct a new glob object with the given options +func newGlob(opts ...GlobOption) *glob { + g := &glob{} + for _, opt := range opts { + opt(g) + } + return g +} + +// WithFailOnIOErrors is an option that can be passed to Glob, GlobWalk, or +// FilepathGlob. If passed, doublestar will abort and return IO errors when +// encountered. Note that if the glob pattern references a path that does not +// exist (such as `nonexistent/path/*`), this is _not_ considered an IO error: +// it is considered a pattern with no matches. +// +func WithFailOnIOErrors() GlobOption { + return func(g *glob) { + g.failOnIOErrors = true + } +} + +// WithFailOnPatternNotExist is an option that can be passed to Glob, GlobWalk, +// or FilepathGlob. If passed, doublestar will abort and return +// ErrPatternNotExist if the pattern references a path that does not exist +// before any meta charcters such as `nonexistent/path/*`. Note that alts (ie, +// `{...}`) are expanded before this check. In other words, a pattern such as +// `{a,b}/*` may fail if either `a` or `b` do not exist but `*/{a,b}` will +// never fail because the star may match nothing. +// +func WithFailOnPatternNotExist() GlobOption { + return func(g *glob) { + g.failOnPatternNotExist = true + } +} + +// WithFilesOnly is an option that can be passed to Glob, GlobWalk, or +// FilepathGlob. If passed, doublestar will only return files that match the +// pattern, not directories. +// +// Note: if combined with the WithNoFollow option, symlinks to directories +// _will_ be included in the result since no attempt is made to follow the +// symlink. +// +func WithFilesOnly() GlobOption { + return func(g *glob) { + g.filesOnly = true + } +} + +// WithNoFollow is an option that can be passed to Glob, GlobWalk, or +// FilepathGlob. If passed, doublestar will not follow symlinks while +// traversing the filesystem. However, due to io/fs's _very_ poor support for +// querying the filesystem about symlinks, there's a caveat here: if part of +// the pattern before any meta characters contains a reference to a symlink, it +// will be followed. For example, a pattern such as `path/to/symlink/*` will be +// followed assuming it is a valid symlink to a directory. However, from this +// same example, a pattern such as `path/to/**` will not traverse the +// `symlink`, nor would `path/*/symlink/*` +// +// Note: if combined with the WithFilesOnly option, symlinks to directories +// _will_ be included in the result since no attempt is made to follow the +// symlink. +// +func WithNoFollow() GlobOption { + return func(g *glob) { + g.noFollow = true + } +} + +// forwardErrIfFailOnIOErrors is used to wrap the return values of I/O +// functions. When failOnIOErrors is enabled, it will return err; otherwise, it +// always returns nil. +// +func (g *glob) forwardErrIfFailOnIOErrors(err error) error { + if g.failOnIOErrors { + return err + } + return nil +} + +// handleErrNotExist handles fs.ErrNotExist errors. If +// WithFailOnPatternNotExist has been enabled and canFail is true, this will +// return ErrPatternNotExist. Otherwise, it will return nil. +// +func (g *glob) handlePatternNotExist(canFail bool) error { + if canFail && g.failOnPatternNotExist { + return ErrPatternNotExist + } + return nil +} + +// Format options for debugging/testing purposes +func (g *glob) GoString() string { + var b strings.Builder + b.WriteString("opts: ") + + hasOpts := false + if g.failOnIOErrors { + b.WriteString("WithFailOnIOErrors") + hasOpts = true + } + if g.failOnPatternNotExist { + if hasOpts { + b.WriteString(", ") + } + b.WriteString("WithFailOnPatternNotExist") + hasOpts = true + } + if g.filesOnly { + if hasOpts { + b.WriteString(", ") + } + b.WriteString("WithFilesOnly") + hasOpts = true + } + if g.noFollow { + if hasOpts { + b.WriteString(", ") + } + b.WriteString("WithNoFollow") + hasOpts = true + } + + if !hasOpts { + b.WriteString("nil") + } + return b.String() +} diff --git a/vendor/github.com/bmatcuk/doublestar/v4/globwalk.go b/vendor/github.com/bmatcuk/doublestar/v4/globwalk.go new file mode 100644 index 000000000..84e764f0e --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/globwalk.go @@ -0,0 +1,414 @@ +package doublestar + +import ( + "errors" + "io/fs" + "path" + "path/filepath" + "strings" +) + +// If returned from GlobWalkFunc, will cause GlobWalk to skip the current +// directory. In other words, if the current path is a directory, GlobWalk will +// not recurse into it. Otherwise, GlobWalk will skip the rest of the current +// directory. +var SkipDir = fs.SkipDir + +// Callback function for GlobWalk(). If the function returns an error, GlobWalk +// will end immediately and return the same error. +type GlobWalkFunc func(path string, d fs.DirEntry) error + +// GlobWalk calls the callback function `fn` for every file matching pattern. +// The syntax of pattern is the same as in Match() and the behavior is the same +// as Glob(), with regard to limitations (such as patterns containing `/./`, +// `/../`, or starting with `/`). The pattern may describe hierarchical names +// such as usr/*/bin/ed. +// +// GlobWalk may have a small performance benefit over Glob if you do not need a +// slice of matches because it can avoid allocating memory for the matches. +// Additionally, GlobWalk gives you access to the `fs.DirEntry` objects for +// each match, and lets you quit early by returning a non-nil error from your +// callback function. Like `io/fs.WalkDir`, if your callback returns `SkipDir`, +// GlobWalk will skip the current directory. This means that if the current +// path _is_ a directory, GlobWalk will not recurse into it. If the current +// path is not a directory, the rest of the parent directory will be skipped. +// +// GlobWalk ignores file system errors such as I/O errors reading directories +// by default. GlobWalk may return ErrBadPattern, reporting that the pattern is +// malformed. +// +// To enable aborting on I/O errors, the WithFailOnIOErrors option can be +// passed. +// +// Additionally, if the callback function `fn` returns an error, GlobWalk will +// exit immediately and return that error. +// +// Like Glob(), this function assumes that your pattern uses `/` as the path +// separator even if that's not correct for your OS (like Windows). If you +// aren't sure if that's the case, you can use filepath.ToSlash() on your +// pattern before calling GlobWalk(). +// +// Note: users should _not_ count on the returned error, +// doublestar.ErrBadPattern, being equal to path.ErrBadPattern. +// +func GlobWalk(fsys fs.FS, pattern string, fn GlobWalkFunc, opts ...GlobOption) error { + if !ValidatePattern(pattern) { + return ErrBadPattern + } + + g := newGlob(opts...) + return g.doGlobWalk(fsys, pattern, true, true, fn) +} + +// Actually execute GlobWalk +// - firstSegment is true if we're in the first segment of the pattern, ie, +// the right-most part where we can match files. If it's false, we're +// somewhere in the middle (or at the beginning) and can only match +// directories since there are path segments above us. +// - beforeMeta is true if we're exploring segments before any meta +// characters, ie, in a pattern such as `path/to/file*.txt`, the `path/to/` +// bit does not contain any meta characters. +func (g *glob) doGlobWalk(fsys fs.FS, pattern string, firstSegment, beforeMeta bool, fn GlobWalkFunc) error { + patternStart := indexMeta(pattern) + if patternStart == -1 { + // pattern doesn't contain any meta characters - does a file matching the + // pattern exist? + // The pattern may contain escaped wildcard characters for an exact path match. + path := unescapeMeta(pattern) + info, pathExists, err := g.exists(fsys, path, beforeMeta) + if pathExists && (!firstSegment || !g.filesOnly || !info.IsDir()) { + err = fn(path, dirEntryFromFileInfo(info)) + if err == SkipDir { + err = nil + } + } + return err + } + + dir := "." + splitIdx := lastIndexSlashOrAlt(pattern) + if splitIdx != -1 { + if pattern[splitIdx] == '}' { + openingIdx := indexMatchedOpeningAlt(pattern[:splitIdx]) + if openingIdx == -1 { + // if there's no matching opening index, technically Match() will treat + // an unmatched `}` as nothing special, so... we will, too! + splitIdx = lastIndexSlash(pattern[:splitIdx]) + if splitIdx != -1 { + dir = pattern[:splitIdx] + pattern = pattern[splitIdx+1:] + } + } else { + // otherwise, we have to handle the alts: + return g.globAltsWalk(fsys, pattern, openingIdx, splitIdx, firstSegment, beforeMeta, fn) + } + } else { + dir = pattern[:splitIdx] + pattern = pattern[splitIdx+1:] + } + } + + // if `splitIdx` is less than `patternStart`, we know `dir` has no meta + // characters. They would be equal if they are both -1, which means `dir` + // will be ".", and we know that doesn't have meta characters either. + if splitIdx <= patternStart { + return g.globDirWalk(fsys, dir, pattern, firstSegment, beforeMeta, fn) + } + + return g.doGlobWalk(fsys, dir, false, beforeMeta, func(p string, d fs.DirEntry) error { + if err := g.globDirWalk(fsys, p, pattern, firstSegment, false, fn); err != nil { + return err + } + return nil + }) +} + +// handle alts in the glob pattern - `openingIdx` and `closingIdx` are the +// indexes of `{` and `}`, respectively +func (g *glob) globAltsWalk(fsys fs.FS, pattern string, openingIdx, closingIdx int, firstSegment, beforeMeta bool, fn GlobWalkFunc) (err error) { + var matches []DirEntryWithFullPath + startIdx := 0 + afterIdx := closingIdx + 1 + splitIdx := lastIndexSlashOrAlt(pattern[:openingIdx]) + if splitIdx == -1 || pattern[splitIdx] == '}' { + // no common prefix + matches, err = g.doGlobAltsWalk(fsys, "", pattern, startIdx, openingIdx, closingIdx, afterIdx, firstSegment, beforeMeta, matches) + if err != nil { + return + } + } else { + // our alts have a common prefix that we can process first + startIdx = splitIdx + 1 + innerBeforeMeta := beforeMeta && !hasMetaExceptAlts(pattern[:splitIdx]) + err = g.doGlobWalk(fsys, pattern[:splitIdx], false, beforeMeta, func(p string, d fs.DirEntry) (e error) { + matches, e = g.doGlobAltsWalk(fsys, p, pattern, startIdx, openingIdx, closingIdx, afterIdx, firstSegment, innerBeforeMeta, matches) + return e + }) + if err != nil { + return + } + } + + skip := "" + for _, m := range matches { + if skip != "" { + // Because matches are sorted, we know that descendants of the skipped + // item must come immediately after the skipped item. If we find an item + // that does not have a prefix matching the skipped item, we know we're + // done skipping. I'm using strings.HasPrefix here because + // filepath.HasPrefix has been marked deprecated (and just calls + // strings.HasPrefix anyway). The reason it's deprecated is because it + // doesn't handle case-insensitive paths, nor does it guarantee that the + // prefix is actually a parent directory. Neither is an issue here: the + // paths come from the system so their cases will match, and we guarantee + // a parent directory by appending a slash to the prefix. + // + // NOTE: m.Path will always use slashes as path separators. + if strings.HasPrefix(m.Path, skip) { + continue + } + skip = "" + } + if err = fn(m.Path, m.Entry); err != nil { + if err == SkipDir { + isDir, err := g.isDir(fsys, "", m.Path, m.Entry) + if err != nil { + return err + } + if isDir { + // append a slash to guarantee `skip` will be treated as a parent dir + skip = m.Path + "/" + } else { + // Dir() calls Clean() which calls FromSlash(), so we need to convert + // back to slashes + skip = filepath.ToSlash(filepath.Dir(m.Path)) + "/" + } + err = nil + continue + } + return + } + } + + return +} + +// runs actual matching for alts +func (g *glob) doGlobAltsWalk(fsys fs.FS, d, pattern string, startIdx, openingIdx, closingIdx, afterIdx int, firstSegment, beforeMeta bool, m []DirEntryWithFullPath) (matches []DirEntryWithFullPath, err error) { + matches = m + matchesLen := len(m) + patIdx := openingIdx + 1 + for patIdx < closingIdx { + nextIdx := indexNextAlt(pattern[patIdx:closingIdx], true) + if nextIdx == -1 { + nextIdx = closingIdx + } else { + nextIdx += patIdx + } + + alt := buildAlt(d, pattern, startIdx, openingIdx, patIdx, nextIdx, afterIdx) + err = g.doGlobWalk(fsys, alt, firstSegment, beforeMeta, func(p string, d fs.DirEntry) error { + // insertion sort, ignoring dups + insertIdx := matchesLen + for insertIdx > 0 && matches[insertIdx-1].Path > p { + insertIdx-- + } + if insertIdx > 0 && matches[insertIdx-1].Path == p { + // dup + return nil + } + + // append to grow the slice, then insert + entry := DirEntryWithFullPath{d, p} + matches = append(matches, entry) + for i := matchesLen; i > insertIdx; i-- { + matches[i] = matches[i-1] + } + matches[insertIdx] = entry + matchesLen++ + + return nil + }) + if err != nil { + return + } + + patIdx = nextIdx + 1 + } + + return +} + +func (g *glob) globDirWalk(fsys fs.FS, dir, pattern string, canMatchFiles, beforeMeta bool, fn GlobWalkFunc) (e error) { + if pattern == "" { + if !canMatchFiles || !g.filesOnly { + // pattern can be an empty string if the original pattern ended in a + // slash, in which case, we should just return dir, but only if it + // actually exists and it's a directory (or a symlink to a directory) + info, isDir, err := g.isPathDir(fsys, dir, beforeMeta) + if err != nil { + return err + } + if isDir { + e = fn(dir, dirEntryFromFileInfo(info)) + if e == SkipDir { + e = nil + } + } + } + return + } + + if pattern == "**" { + // `**` can match *this* dir + info, dirExists, err := g.exists(fsys, dir, beforeMeta) + if err != nil { + return err + } + if !dirExists || !info.IsDir() { + return nil + } + if !canMatchFiles || !g.filesOnly { + if e = fn(dir, dirEntryFromFileInfo(info)); e != nil { + if e == SkipDir { + e = nil + } + return + } + } + return g.globDoubleStarWalk(fsys, dir, canMatchFiles, fn) + } + + dirs, err := fs.ReadDir(fsys, dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return g.handlePatternNotExist(beforeMeta) + } + return g.forwardErrIfFailOnIOErrors(err) + } + + var matched bool + for _, info := range dirs { + name := info.Name() + matched, e = matchWithSeparator(pattern, name, '/', false) + if e != nil { + return + } + if matched { + matched = canMatchFiles + if !matched || g.filesOnly { + matched, e = g.isDir(fsys, dir, name, info) + if e != nil { + return e + } + if canMatchFiles { + // if we're here, it's because g.filesOnly + // is set and we don't want directories + matched = !matched + } + } + if matched { + if e = fn(path.Join(dir, name), info); e != nil { + if e == SkipDir { + e = nil + } + return + } + } + } + } + + return +} + +// recursively walk files/directories in a directory +func (g *glob) globDoubleStarWalk(fsys fs.FS, dir string, canMatchFiles bool, fn GlobWalkFunc) (e error) { + dirs, err := fs.ReadDir(fsys, dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // This function is only ever called after we know the top-most directory + // exists, so, if we ever get here, we know we'll never return + // ErrPatternNotExist. + return nil + } + return g.forwardErrIfFailOnIOErrors(err) + } + + for _, info := range dirs { + name := info.Name() + isDir, err := g.isDir(fsys, dir, name, info) + if err != nil { + return err + } + + if isDir { + p := path.Join(dir, name) + if !canMatchFiles || !g.filesOnly { + // `**` can match *this* dir, so add it + if e = fn(p, info); e != nil { + if e == SkipDir { + e = nil + continue + } + return + } + } + if e = g.globDoubleStarWalk(fsys, p, canMatchFiles, fn); e != nil { + return + } + } else if canMatchFiles { + if e = fn(path.Join(dir, name), info); e != nil { + if e == SkipDir { + e = nil + } + return + } + } + } + + return +} + +type DirEntryFromFileInfo struct { + fi fs.FileInfo +} + +func (d *DirEntryFromFileInfo) Name() string { + return d.fi.Name() +} + +func (d *DirEntryFromFileInfo) IsDir() bool { + return d.fi.IsDir() +} + +func (d *DirEntryFromFileInfo) Type() fs.FileMode { + return d.fi.Mode().Type() +} + +func (d *DirEntryFromFileInfo) Info() (fs.FileInfo, error) { + return d.fi, nil +} + +func dirEntryFromFileInfo(fi fs.FileInfo) fs.DirEntry { + return &DirEntryFromFileInfo{fi} +} + +type DirEntryWithFullPath struct { + Entry fs.DirEntry + Path string +} + +func hasMetaExceptAlts(s string) bool { + var c byte + l := len(s) + for i := 0; i < l; i++ { + c = s[i] + if c == '*' || c == '?' || c == '[' { + return true + } else if c == '\\' { + // skip next byte + i++ + } + } + return false +} diff --git a/vendor/github.com/bmatcuk/doublestar/v4/match.go b/vendor/github.com/bmatcuk/doublestar/v4/match.go new file mode 100644 index 000000000..6581d9986 --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/match.go @@ -0,0 +1,376 @@ +package doublestar + +import ( + "path/filepath" + "unicode/utf8" +) + +// Match reports whether name matches the shell pattern. +// The pattern syntax is: +// +// pattern: +// { term } +// term: +// '*' matches any sequence of non-path-separators +// '/**/' matches zero or more directories +// '?' matches any single non-path-separator character +// '[' [ '^' '!' ] { character-range } ']' +// character class (must be non-empty) +// starting with `^` or `!` negates the class +// '{' { term } [ ',' { term } ... ] '}' +// alternatives +// c matches character c (c != '*', '?', '\\', '[') +// '\\' c matches character c +// +// character-range: +// c matches character c (c != '\\', '-', ']') +// '\\' c matches character c +// lo '-' hi matches character c for lo <= c <= hi +// +// Match returns true if `name` matches the file name `pattern`. `name` and +// `pattern` are split on forward slash (`/`) characters and may be relative or +// absolute. +// +// Match requires pattern to match all of name, not just a substring. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// A doublestar (`**`) should appear surrounded by path separators such as +// `/**/`. A mid-pattern doublestar (`**`) behaves like bash's globstar +// option: a pattern such as `path/to/**.txt` would return the same results as +// `path/to/*.txt`. The pattern you're looking for is `path/to/**/*.txt`. +// +// Note: this is meant as a drop-in replacement for path.Match() which +// always uses '/' as the path separator. If you want to support systems +// which use a different path separator (such as Windows), what you want +// is PathMatch(). Alternatively, you can run filepath.ToSlash() on both +// pattern and name and then use this function. +// +// Note: users should _not_ count on the returned error, +// doublestar.ErrBadPattern, being equal to path.ErrBadPattern. +// +func Match(pattern, name string) (bool, error) { + return matchWithSeparator(pattern, name, '/', true) +} + +// PathMatch returns true if `name` matches the file name `pattern`. The +// difference between Match and PathMatch is that PathMatch will automatically +// use your system's path separator to split `name` and `pattern`. On systems +// where the path separator is `'\'`, escaping will be disabled. +// +// Note: this is meant as a drop-in replacement for filepath.Match(). It +// assumes that both `pattern` and `name` are using the system's path +// separator. If you can't be sure of that, use filepath.ToSlash() on both +// `pattern` and `name`, and then use the Match() function instead. +// +func PathMatch(pattern, name string) (bool, error) { + return matchWithSeparator(pattern, name, filepath.Separator, true) +} + +func matchWithSeparator(pattern, name string, separator rune, validate bool) (matched bool, err error) { + return doMatchWithSeparator(pattern, name, separator, validate, -1, -1, -1, -1, 0, 0) +} + +func doMatchWithSeparator(pattern, name string, separator rune, validate bool, doublestarPatternBacktrack, doublestarNameBacktrack, starPatternBacktrack, starNameBacktrack, patIdx, nameIdx int) (matched bool, err error) { + patLen := len(pattern) + nameLen := len(name) + startOfSegment := true +MATCH: + for nameIdx < nameLen { + if patIdx < patLen { + switch pattern[patIdx] { + case '*': + if patIdx++; patIdx < patLen && pattern[patIdx] == '*' { + // doublestar - must begin with a path separator, otherwise we'll + // treat it like a single star like bash + patIdx++ + if startOfSegment { + if patIdx >= patLen { + // pattern ends in `/**`: return true + return true, nil + } + + // doublestar must also end with a path separator, otherwise we're + // just going to treat the doublestar as a single star like bash + patRune, patRuneLen := utf8.DecodeRuneInString(pattern[patIdx:]) + if patRune == separator { + patIdx += patRuneLen + + doublestarPatternBacktrack = patIdx + doublestarNameBacktrack = nameIdx + starPatternBacktrack = -1 + starNameBacktrack = -1 + continue + } + } + } + startOfSegment = false + + starPatternBacktrack = patIdx + starNameBacktrack = nameIdx + continue + + case '?': + startOfSegment = false + nameRune, nameRuneLen := utf8.DecodeRuneInString(name[nameIdx:]) + if nameRune == separator { + // `?` cannot match the separator + break + } + + patIdx++ + nameIdx += nameRuneLen + continue + + case '[': + startOfSegment = false + if patIdx++; patIdx >= patLen { + // class didn't end + return false, ErrBadPattern + } + nameRune, nameRuneLen := utf8.DecodeRuneInString(name[nameIdx:]) + + matched := false + negate := pattern[patIdx] == '!' || pattern[patIdx] == '^' + if negate { + patIdx++ + } + + if patIdx >= patLen || pattern[patIdx] == ']' { + // class didn't end or empty character class + return false, ErrBadPattern + } + + last := utf8.MaxRune + for patIdx < patLen && pattern[patIdx] != ']' { + patRune, patRuneLen := utf8.DecodeRuneInString(pattern[patIdx:]) + patIdx += patRuneLen + + // match a range + if last < utf8.MaxRune && patRune == '-' && patIdx < patLen && pattern[patIdx] != ']' { + if pattern[patIdx] == '\\' { + // next character is escaped + patIdx++ + } + patRune, patRuneLen = utf8.DecodeRuneInString(pattern[patIdx:]) + patIdx += patRuneLen + + if last <= nameRune && nameRune <= patRune { + matched = true + break + } + + // didn't match range - reset `last` + last = utf8.MaxRune + continue + } + + // not a range - check if the next rune is escaped + if patRune == '\\' { + patRune, patRuneLen = utf8.DecodeRuneInString(pattern[patIdx:]) + patIdx += patRuneLen + } + + // check if the rune matches + if patRune == nameRune { + matched = true + break + } + + // no matches yet + last = patRune + } + + if matched == negate { + // failed to match - if we reached the end of the pattern, that means + // we never found a closing `]` + if patIdx >= patLen { + return false, ErrBadPattern + } + break + } + + closingIdx := indexUnescapedByte(pattern[patIdx:], ']', true) + if closingIdx == -1 { + // no closing `]` + return false, ErrBadPattern + } + + patIdx += closingIdx + 1 + nameIdx += nameRuneLen + continue + + case '{': + startOfSegment = false + beforeIdx := patIdx + patIdx++ + closingIdx := indexMatchedClosingAlt(pattern[patIdx:], separator != '\\') + if closingIdx == -1 { + // no closing `}` + return false, ErrBadPattern + } + closingIdx += patIdx + + for { + commaIdx := indexNextAlt(pattern[patIdx:closingIdx], separator != '\\') + if commaIdx == -1 { + break + } + commaIdx += patIdx + + result, err := doMatchWithSeparator(pattern[:beforeIdx]+pattern[patIdx:commaIdx]+pattern[closingIdx+1:], name, separator, validate, doublestarPatternBacktrack, doublestarNameBacktrack, starPatternBacktrack, starNameBacktrack, beforeIdx, nameIdx) + if result || err != nil { + return result, err + } + + patIdx = commaIdx + 1 + } + return doMatchWithSeparator(pattern[:beforeIdx]+pattern[patIdx:closingIdx]+pattern[closingIdx+1:], name, separator, validate, doublestarPatternBacktrack, doublestarNameBacktrack, starPatternBacktrack, starNameBacktrack, beforeIdx, nameIdx) + + case '\\': + if separator != '\\' { + // next rune is "escaped" in the pattern - literal match + if patIdx++; patIdx >= patLen { + // pattern ended + return false, ErrBadPattern + } + } + fallthrough + + default: + patRune, patRuneLen := utf8.DecodeRuneInString(pattern[patIdx:]) + nameRune, nameRuneLen := utf8.DecodeRuneInString(name[nameIdx:]) + if patRune != nameRune { + if separator != '\\' && patIdx > 0 && pattern[patIdx-1] == '\\' { + // if this rune was meant to be escaped, we need to move patIdx + // back to the backslash before backtracking or validating below + patIdx-- + } + break + } + + patIdx += patRuneLen + nameIdx += nameRuneLen + startOfSegment = patRune == separator + continue + } + } + + if starPatternBacktrack >= 0 { + // `*` backtrack, but only if the `name` rune isn't the separator + nameRune, nameRuneLen := utf8.DecodeRuneInString(name[starNameBacktrack:]) + if nameRune != separator { + starNameBacktrack += nameRuneLen + patIdx = starPatternBacktrack + nameIdx = starNameBacktrack + startOfSegment = false + continue + } + } + + if doublestarPatternBacktrack >= 0 { + // `**` backtrack, advance `name` past next separator + nameIdx = doublestarNameBacktrack + for nameIdx < nameLen { + nameRune, nameRuneLen := utf8.DecodeRuneInString(name[nameIdx:]) + nameIdx += nameRuneLen + if nameRune == separator { + doublestarNameBacktrack = nameIdx + patIdx = doublestarPatternBacktrack + startOfSegment = true + continue MATCH + } + } + } + + if validate && patIdx < patLen && !doValidatePattern(pattern[patIdx:], separator) { + return false, ErrBadPattern + } + return false, nil + } + + if nameIdx < nameLen { + // we reached the end of `pattern` before the end of `name` + return false, nil + } + + // we've reached the end of `name`; we've successfully matched if we've also + // reached the end of `pattern`, or if the rest of `pattern` can match a + // zero-length string + return isZeroLengthPattern(pattern[patIdx:], separator) +} + +func isZeroLengthPattern(pattern string, separator rune) (ret bool, err error) { + // `/**` is a special case - a pattern such as `path/to/a/**` *should* match + // `path/to/a` because `a` might be a directory + if pattern == "" || pattern == "*" || pattern == "**" || pattern == string(separator)+"**" { + return true, nil + } + + if pattern[0] == '{' { + closingIdx := indexMatchedClosingAlt(pattern[1:], separator != '\\') + if closingIdx == -1 { + // no closing '}' + return false, ErrBadPattern + } + closingIdx += 1 + + patIdx := 1 + for { + commaIdx := indexNextAlt(pattern[patIdx:closingIdx], separator != '\\') + if commaIdx == -1 { + break + } + commaIdx += patIdx + + ret, err = isZeroLengthPattern(pattern[patIdx:commaIdx]+pattern[closingIdx+1:], separator) + if ret || err != nil { + return + } + + patIdx = commaIdx + 1 + } + return isZeroLengthPattern(pattern[patIdx:closingIdx]+pattern[closingIdx+1:], separator) + } + + // no luck - validate the rest of the pattern + if !doValidatePattern(pattern, separator) { + return false, ErrBadPattern + } + return false, nil +} + +// Finds the index of the first unescaped byte `c`, or negative 1. +func indexUnescapedByte(s string, c byte, allowEscaping bool) int { + l := len(s) + for i := 0; i < l; i++ { + if allowEscaping && s[i] == '\\' { + // skip next byte + i++ + } else if s[i] == c { + return i + } + } + return -1 +} + +// Assuming the byte before the beginning of `s` is an opening `{`, this +// function will find the index of the matching `}`. That is, it'll skip over +// any nested `{}` and account for escaping +func indexMatchedClosingAlt(s string, allowEscaping bool) int { + alts := 1 + l := len(s) + for i := 0; i < l; i++ { + if allowEscaping && s[i] == '\\' { + // skip next byte + i++ + } else if s[i] == '{' { + alts++ + } else if s[i] == '}' { + if alts--; alts == 0 { + return i + } + } + } + return -1 +} diff --git a/vendor/github.com/bmatcuk/doublestar/v4/utils.go b/vendor/github.com/bmatcuk/doublestar/v4/utils.go new file mode 100644 index 000000000..0ab1dc98f --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/utils.go @@ -0,0 +1,147 @@ +package doublestar + +import ( + "errors" + "os" + "path" + "path/filepath" + "strings" +) + +// SplitPattern is a utility function. Given a pattern, SplitPattern will +// return two strings: the first string is everything up to the last slash +// (`/`) that appears _before_ any unescaped "meta" characters (ie, `*?[{`). +// The second string is everything after that slash. For example, given the +// pattern: +// +// ../../path/to/meta*/** +// ^----------- split here +// +// SplitPattern returns "../../path/to" and "meta*/**". This is useful for +// initializing os.DirFS() to call Glob() because Glob() will silently fail if +// your pattern includes `/./` or `/../`. For example: +// +// base, pattern := SplitPattern("../../path/to/meta*/**") +// fsys := os.DirFS(base) +// matches, err := Glob(fsys, pattern) +// +// If SplitPattern cannot find somewhere to split the pattern (for example, +// `meta*/**`), it will return "." and the unaltered pattern (`meta*/**` in +// this example). +// +// Of course, it is your responsibility to decide if the returned base path is +// "safe" in the context of your application. Perhaps you could use Match() to +// validate against a list of approved base directories? +// +func SplitPattern(p string) (base, pattern string) { + base = "." + pattern = p + + splitIdx := -1 + for i := 0; i < len(p); i++ { + c := p[i] + if c == '\\' { + i++ + } else if c == '/' { + splitIdx = i + } else if c == '*' || c == '?' || c == '[' || c == '{' { + break + } + } + + if splitIdx == 0 { + return "/", p[1:] + } else if splitIdx > 0 { + return p[:splitIdx], p[splitIdx+1:] + } + + return +} + +// FilepathGlob returns the names of all files matching pattern or nil if there +// is no matching file. The syntax of pattern is the same as in Match(). The +// pattern may describe hierarchical names such as usr/*/bin/ed. +// +// FilepathGlob ignores file system errors such as I/O errors reading +// directories by default. The only possible returned error is ErrBadPattern, +// reporting that the pattern is malformed. +// +// To enable aborting on I/O errors, the WithFailOnIOErrors option can be +// passed. +// +// Note: FilepathGlob is a convenience function that is meant as a drop-in +// replacement for `path/filepath.Glob()` for users who don't need the +// complication of io/fs. Basically, it: +// - Runs `filepath.Clean()` and `ToSlash()` on the pattern +// - Runs `SplitPattern()` to get a base path and a pattern to Glob +// - Creates an FS object from the base path and `Glob()s` on the pattern +// - Joins the base path with all of the matches from `Glob()` +// +// Returned paths will use the system's path separator, just like +// `filepath.Glob()`. +// +// Note: the returned error doublestar.ErrBadPattern is not equal to +// filepath.ErrBadPattern. +// +func FilepathGlob(pattern string, opts ...GlobOption) (matches []string, err error) { + pattern = filepath.Clean(pattern) + pattern = filepath.ToSlash(pattern) + base, f := SplitPattern(pattern) + if f == "" || f == "." || f == ".." { + // some special cases to match filepath.Glob behavior + if !ValidatePathPattern(pattern) { + return nil, ErrBadPattern + } + + if filepath.Separator != '\\' { + pattern = unescapeMeta(pattern) + } + + if _, err = os.Lstat(pattern); err != nil { + g := newGlob(opts...) + if errors.Is(err, os.ErrNotExist) { + return nil, g.handlePatternNotExist(true) + } + return nil, g.forwardErrIfFailOnIOErrors(err) + } + return []string{filepath.FromSlash(pattern)}, nil + } + + fs := os.DirFS(base) + if matches, err = Glob(fs, f, opts...); err != nil { + return nil, err + } + for i := range matches { + // use path.Join because we used ToSlash above to ensure our paths are made + // of forward slashes, no matter what the system uses + matches[i] = filepath.FromSlash(path.Join(base, matches[i])) + } + return +} + +// Finds the next comma, but ignores any commas that appear inside nested `{}`. +// Assumes that each opening bracket has a corresponding closing bracket. +func indexNextAlt(s string, allowEscaping bool) int { + alts := 1 + l := len(s) + for i := 0; i < l; i++ { + if allowEscaping && s[i] == '\\' { + // skip next byte + i++ + } else if s[i] == '{' { + alts++ + } else if s[i] == '}' { + alts-- + } else if s[i] == ',' && alts == 1 { + return i + } + } + return -1 +} + +var metaReplacer = strings.NewReplacer("\\*", "*", "\\?", "?", "\\[", "[", "\\]", "]", "\\{", "{", "\\}", "}") + +// Unescapes meta characters (*?[]{}) +func unescapeMeta(pattern string) string { + return metaReplacer.Replace(pattern) +} diff --git a/vendor/github.com/bmatcuk/doublestar/v4/validate.go b/vendor/github.com/bmatcuk/doublestar/v4/validate.go new file mode 100644 index 000000000..c689b9eba --- /dev/null +++ b/vendor/github.com/bmatcuk/doublestar/v4/validate.go @@ -0,0 +1,82 @@ +package doublestar + +import "path/filepath" + +// Validate a pattern. Patterns are validated while they run in Match(), +// PathMatch(), and Glob(), so, you normally wouldn't need to call this. +// However, there are cases where this might be useful: for example, if your +// program allows a user to enter a pattern that you'll run at a later time, +// you might want to validate it. +// +// ValidatePattern assumes your pattern uses '/' as the path separator. +// +func ValidatePattern(s string) bool { + return doValidatePattern(s, '/') +} + +// Like ValidatePattern, only uses your OS path separator. In other words, use +// ValidatePattern if you would normally use Match() or Glob(). Use +// ValidatePathPattern if you would normally use PathMatch(). Keep in mind, +// Glob() requires '/' separators, even if your OS uses something else. +// +func ValidatePathPattern(s string) bool { + return doValidatePattern(s, filepath.Separator) +} + +func doValidatePattern(s string, separator rune) bool { + altDepth := 0 + l := len(s) +VALIDATE: + for i := 0; i < l; i++ { + switch s[i] { + case '\\': + if separator != '\\' { + // skip the next byte - return false if there is no next byte + if i++; i >= l { + return false + } + } + continue + + case '[': + if i++; i >= l { + // class didn't end + return false + } + if s[i] == '^' || s[i] == '!' { + i++ + } + if i >= l || s[i] == ']' { + // class didn't end or empty character class + return false + } + + for ; i < l; i++ { + if separator != '\\' && s[i] == '\\' { + i++ + } else if s[i] == ']' { + // looks good + continue VALIDATE + } + } + + // class didn't end + return false + + case '{': + altDepth++ + continue + + case '}': + if altDepth == 0 { + // alt end without a corresponding start + return false + } + altDepth-- + continue + } + } + + // valid as long as all alts are closed + return altDepth == 0 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ee367c62b..37f638343 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -245,6 +245,9 @@ github.com/aws/smithy-go/waiter # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile +# github.com/bmatcuk/doublestar/v4 v4.6.0 +## explicit; go 1.16 +github.com/bmatcuk/doublestar/v4 # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2