Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 17de3f8

Browse files
jphastingsjphastings
committedJul 9, 2023
feat!: Matches required query parameters
Code now parses and `MatchAndExpandPlaceholders` for query parameters as well as paths. Placeholders are shared between the two. This commit codifies some of the previously implicit edgecases, particularly around duplicate placeholders. Closes #20
1 parent 00c708c commit 17de3f8

File tree

4 files changed

+492
-27
lines changed

4 files changed

+492
-27
lines changed
 

‎README.md

+6-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ This is a parser for the IPFS Web Gateway's `_redirects` file format.
77
Follow specification work at https://github.com/ipfs/specs/pull/290
88

99
## Format
10-
Currently only supports `from`, `to` and `status`.
10+
Currently only supports `from`, `fromQuery`, `to` and `status`.
1111

1212
```
13-
from to [status]
13+
from [fromQuery [fromQuery]] to [status]
1414
```
1515

1616
## Example
@@ -39,6 +39,10 @@ from to [status]
3939

4040
# Single page app rewrite (SPA, PWA)
4141
/* /index.html 200
42+
43+
# Query parameter rewrite
44+
/thing type=:type /things/:type.html 200
45+
/thing /things.html 200
4246
```
4347

4448
## Notes for contributors

‎redirects.go

+130-19
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"io"
88
"net/url"
9+
"regexp"
910
"strconv"
1011
"strings"
1112

@@ -21,6 +22,12 @@ type Rule struct {
2122
// From is the path which is matched to perform the rule.
2223
From string
2324

25+
// FromQuery is the set of required query parameters which
26+
// must be present to perform the rule.
27+
// A string without a preceding colon requires that query parameter is this exact value.
28+
// A string with a preceding colon will match any value, and provide it as a placeholder.
29+
FromQuery map[string]string
30+
2431
// To is the destination which may be relative, or absolute
2532
// in order to proxy the request to another URL.
2633
To string
@@ -51,37 +58,70 @@ func (r *Rule) IsProxy() bool {
5158

5259
// MatchAndExpandPlaceholders expands placeholders in `r.To` and returns true if the provided path matches.
5360
// Otherwise it returns false.
54-
func (r *Rule) MatchAndExpandPlaceholders(urlPath string) bool {
61+
func (r *Rule) MatchAndExpandPlaceholders(urlPath string, urlParams url.Values) bool {
5562
// get rule.From, trim trailing slash, ...
5663
fromPath := urlpath.New(strings.TrimSuffix(r.From, "/"))
5764
match, ok := fromPath.Match(urlPath)
58-
5965
if !ok {
6066
return false
6167
}
6268

63-
// We have a match! Perform substitution and return the updated rule
69+
placeholders := match.Params
70+
placeholders["splat"] = match.Trailing
71+
if !matchParams(r.FromQuery, urlParams, placeholders) {
72+
return false
73+
}
74+
75+
// We have a match! Perform substitution and return the updated rule
6476
toPath := r.To
65-
toPath = replacePlaceholders(toPath, match)
66-
toPath = replaceSplat(toPath, match)
77+
toPath = replacePlaceholders(toPath, placeholders)
78+
79+
// There's a placeholder unsupplied somewhere
80+
if strings.Contains(toPath, ":") {
81+
return false
82+
}
6783

6884
r.To = toPath
6985

7086
return true
7187
}
7288

73-
func replacePlaceholders(to string, match urlpath.Match) string {
74-
if len(match.Params) > 0 {
75-
for key, value := range match.Params {
76-
to = strings.ReplaceAll(to, ":"+key, value)
77-
}
89+
func replacePlaceholders(to string, placeholders map[string]string) string {
90+
if len(placeholders) == 0 {
91+
return to
92+
}
93+
94+
for key, value := range placeholders {
95+
to = strings.ReplaceAll(to, ":"+key, value)
7896
}
7997

8098
return to
8199
}
82100

83-
func replaceSplat(to string, match urlpath.Match) string {
84-
return strings.ReplaceAll(to, ":splat", match.Trailing)
101+
func replaceSplat(to string, splat string) string {
102+
return strings.ReplaceAll(to, ":splat", splat)
103+
}
104+
105+
func matchParams(fromQuery map[string]string, urlParams url.Values, placeholders map[string]string) bool {
106+
for neededK, neededV := range fromQuery {
107+
haveVs, ok := urlParams[neededK]
108+
if !ok {
109+
return false
110+
}
111+
112+
if isPlaceholder(neededV) {
113+
if _, ok := placeholders[neededV[1:]]; !ok {
114+
placeholders[neededV[1:]] = haveVs[0]
115+
}
116+
continue
117+
}
118+
119+
if !contains(haveVs, neededV) {
120+
return false
121+
}
122+
}
123+
124+
return true
85125
}
86126

87127
// Must parse utility.
@@ -124,10 +164,6 @@ func Parse(r io.Reader) (rules []Rule, err error) {
124164
return nil, fmt.Errorf("missing 'to' path")
125165
}
126166

127-
if len(fields) > 3 {
128-
return nil, fmt.Errorf("must match format 'from to [status]'")
129-
}
130-
131167
// implicit status
132168
rule := Rule{Status: 301}
133169

@@ -138,23 +174,42 @@ func Parse(r io.Reader) (rules []Rule, err error) {
138174
}
139175
rule.From = from
140176

177+
hasStatus := isLikelyStatusCode(fields[len(fields)-1])
178+
toIndex := len(fields) - 1
179+
if hasStatus {
180+
toIndex = len(fields) - 2
181+
}
182+
141183
// to (must parse as an absolute path or an URL)
142-
to, err := parseTo(fields[1])
184+
to, err := parseTo(fields[toIndex])
143185
if err != nil {
144186
return nil, errors.Wrapf(err, "parsing 'to'")
145187
}
146188
rule.To = to
147189

148190
// status
149-
if len(fields) > 2 {
150-
code, err := parseStatus(fields[2])
191+
if hasStatus {
192+
code, err := parseStatus(fields[len(fields)-1])
151193
if err != nil {
152194
return nil, errors.Wrapf(err, "parsing status %q", fields[2])
153195
}
154196

155197
rule.Status = code
156198
}
157199

200+
// from query
201+
if toIndex > 1 {
202+
rule.FromQuery = make(map[string]string)
203+
204+
for i := 1; i < toIndex; i++ {
205+
key, value, err := parseFromQuery(fields[i])
206+
if err != nil {
207+
return nil, errors.Wrapf(err, "parsing 'fromQuery'")
208+
}
209+
rule.FromQuery[key] = value
210+
}
211+
}
212+
158213
rules = append(rules, rule)
159214
}
160215

@@ -194,6 +249,46 @@ func parseFrom(s string) (string, error) {
194249
return s, nil
195250
}
196251

252+
func parseFromQuery(s string) (string, string, error) {
253+
params, err := url.ParseQuery(s)
254+
if err != nil {
255+
return "", "", err
256+
}
257+
if len(params) != 1 {
258+
return "", "", fmt.Errorf("separate different fromQuery arguments with a space")
259+
}
260+
261+
var key string
262+
var val []string
263+
// We know there's only 1, but we don't know the key to access it
264+
for k, v := range params {
265+
key = k
266+
val = v
267+
}
268+
269+
if url.QueryEscape(key) != key {
270+
return "", "", fmt.Errorf("fromQuery key must be URL encoded")
271+
}
272+
273+
if len(val) > 1 {
274+
return "", "", fmt.Errorf("separate different fromQuery arguments with a space")
275+
}
276+
277+
ignorePlaceholders := val[0]
278+
if isPlaceholder(val[0]) {
279+
ignorePlaceholders = ignorePlaceholders[1:]
280+
}
281+
282+
if url.QueryEscape(ignorePlaceholders) != ignorePlaceholders {
283+
return "", "", fmt.Errorf("fromQuery val must be URL encoded")
284+
}
285+
return key, val[0], nil
286+
}
287+
288+
func isPlaceholder(s string) bool {
289+
return strings.HasPrefix(s, ":")
290+
}
291+
197292
func parseTo(s string) (string, error) {
198293
// confirm value is within URL path spec
199294
u, err := url.Parse(s)
@@ -211,6 +306,13 @@ func parseTo(s string) (string, error) {
211306
return s, nil
212307
}
213308

309+
var likeStatusCode = regexp.MustCompile(`^\d{1,3}!?$`)
310+
311+
// isLikelyStatusCode returns true if the given string is likely to be a status code.
312+
func isLikelyStatusCode(s string) bool {
313+
return likeStatusCode.MatchString(s)
314+
}
315+
214316
// parseStatus returns the status code.
215317
func parseStatus(s string) (code int, err error) {
216318
if strings.HasSuffix(s, "!") {
@@ -237,3 +339,12 @@ func isValidStatusCode(status int) bool {
237339
}
238340
return false
239341
}
342+
343+
func contains(arr []string, s string) bool {
344+
for _, a := range arr {
345+
if a == s {
346+
return true
347+
}
348+
}
349+
return false
350+
}

‎redirects_example_test.go

+70
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,18 @@ func Example() {
3030
3131
# Proxying
3232
/api/* https://api.example.com/:splat 200
33+
34+
# Query parameters
35+
/things type=photos /photos.html 200
36+
/things type= /empty.html 200
37+
/things type=:thing /things/:thing.html 200
38+
/things /things.html 200
39+
40+
# Multiple query parameters
41+
/stuff type=lost name=:name other=:ignore /other-stuff/:name.html 200
42+
43+
# Query parameters with implicit 301
44+
/items id=:id /items/:id.html
3345
`))
3446

3547
enc := json.NewEncoder(os.Stdout)
@@ -39,53 +51,111 @@ func Example() {
3951
// [
4052
// {
4153
// "From": "/home",
54+
// "FromQuery": null,
4255
// "To": "/",
4356
// "Status": 301
4457
// },
4558
// {
4659
// "From": "/blog/my-post.php",
60+
// "FromQuery": null,
4761
// "To": "/blog/my-post",
4862
// "Status": 301
4963
// },
5064
// {
5165
// "From": "/news",
66+
// "FromQuery": null,
5267
// "To": "/blog",
5368
// "Status": 301
5469
// },
5570
// {
5671
// "From": "/google",
72+
// "FromQuery": null,
5773
// "To": "https://www.google.com",
5874
// "Status": 301
5975
// },
6076
// {
6177
// "From": "/home",
78+
// "FromQuery": null,
6279
// "To": "/",
6380
// "Status": 301
6481
// },
6582
// {
6683
// "From": "/my-redirect",
84+
// "FromQuery": null,
6785
// "To": "/",
6886
// "Status": 302
6987
// },
7088
// {
7189
// "From": "/pass-through",
90+
// "FromQuery": null,
7291
// "To": "/index.html",
7392
// "Status": 200
7493
// },
7594
// {
7695
// "From": "/ecommerce",
96+
// "FromQuery": null,
7797
// "To": "/store-closed",
7898
// "Status": 404
7999
// },
80100
// {
81101
// "From": "/*",
102+
// "FromQuery": null,
82103
// "To": "/index.html",
83104
// "Status": 200
84105
// },
85106
// {
86107
// "From": "/api/*",
108+
// "FromQuery": null,
87109
// "To": "https://api.example.com/:splat",
88110
// "Status": 200
111+
// },
112+
// {
113+
// "From": "/things",
114+
// "FromQuery": {
115+
// "type": "photos"
116+
// },
117+
// "To": "/photos.html",
118+
// "Status": 200
119+
// },
120+
// {
121+
// "From": "/things",
122+
// "FromQuery": {
123+
// "type": ""
124+
// },
125+
// "To": "/empty.html",
126+
// "Status": 200
127+
// },
128+
// {
129+
// "From": "/things",
130+
// "FromQuery": {
131+
// "type": ":thing"
132+
// },
133+
// "To": "/things/:thing.html",
134+
// "Status": 200
135+
// },
136+
// {
137+
// "From": "/things",
138+
// "FromQuery": null,
139+
// "To": "/things.html",
140+
// "Status": 200
141+
// },
142+
// {
143+
// "From": "/stuff",
144+
// "FromQuery": {
145+
// "name": ":name",
146+
// "other": ":ignore",
147+
// "type": "lost"
148+
// },
149+
// "To": "/other-stuff/:name.html",
150+
// "Status": 200
151+
// },
152+
// {
153+
// "From": "/items",
154+
// "FromQuery": {
155+
// "id": ":id"
156+
// },
157+
// "To": "/items/:id.html",
158+
// "Status": 301
89159
// }
90160
// ]
91161
}

‎redirects_test.go

+286-6
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,28 @@ func TestParse(t *testing.T) {
9898
assert.Error(t, err)
9999
assert.Contains(t, err.Error(), "redirects file size cannot exceed")
100100
})
101+
102+
t.Run("with fromQuery arguments", func(t *testing.T) {
103+
rules, err := ParseString(`
104+
/fixed type=type /type.html
105+
/dynamic type=:type /type-:type.html
106+
/empty type= /empty-type.html
107+
/any type=:ignore /any-type.html
108+
/multi a=a b=:b c= d /multi-:b.html
109+
/fixed200 type=type /type.html 200
110+
/dynamic200 type=:type /type-:type.html 200
111+
/empty200 type= /empty-type.html 200
112+
/any200 type=:ignore /any-type.html 200
113+
/multi200 a=a b=:b c= d /multi-:b.html 200
114+
`)
115+
116+
assert.NoError(t, err)
117+
assert.Len(t, rules, 10)
118+
assert.Equal(t, "type", rules[0].FromQuery["type"])
119+
assert.Equal(t, ":type", rules[1].FromQuery["type"])
120+
assert.Equal(t, "", rules[2].FromQuery["type"])
121+
assert.Equal(t, ":ignore", rules[3].FromQuery["type"])
122+
})
101123
}
102124

103125
func FuzzParse(f *testing.F) {
@@ -108,7 +130,12 @@ func FuzzParse(f *testing.F) {
108130
"/%C4%85 /ę 301\n",
109131
"#/a \n\n/b",
110132
"/a200 /b200 200\n/a301 /b301 301\n/a302 /b302 302\n/a303 /b303 303\n/a307 /b307 307\n/a308 /b308 308\n/a404 /b404 404\n/a410 /b410 410\n/a451 /b451 451\n",
111-
"hello\n", "/redirect-one /one.html\r\n/200-index /index.html 200\r\n", "a b 2\nc d 42", "/a/*/b blah", "/from https://example.com 200\n/a/:blah/yeah /b/:blah/yeah"}
133+
"hello\n", "/redirect-one /one.html\r\n/200-index /index.html 200\r\n", "a b 2\nc d 42", "/a/*/b blah", "/from https://example.com 200\n/a/:blah/yeah /b/:blah/yeah",
134+
"/fixed-val val=val /to\n", "/dynamic-val val=:val /to/:val\n", "/empty-val val= /to\n", "/any-val val /to\n",
135+
"/fixed-val val=val /to 200\n/dynamic-val val=:val /to/:val 301\n/empty-val val= /to 404\n/any-val val /to 302\n",
136+
"/multi-query val1=val1 val2=:val2 val3= val4 /to/:val2\n/multi-query2 val1=val1 val2=:val2 val3= val4 /to/:val2 302\n",
137+
"/bad-syntax1 val=a&val=b /to\n", "/bad-syntax2 val=a&val2=b /to 302\n", "/a ^&notparams /b\n", "/bad-status type=:type /to 3oo\n", "/bad-chars :type=whatever /to\n", "/bad-chars type=what:ever /to\n",
138+
}
112139
for _, tc := range testcases {
113140
f.Add([]byte(tc))
114141
}
@@ -154,6 +181,21 @@ func FuzzParse(f *testing.F) {
154181
t.Errorf("should error for 'to' URL with scheme other than safelisted ones: url=%q, scheme=%q, orig=%q", to, to.Scheme, orig)
155182
}
156183
}
184+
185+
for key, val := range r.FromQuery {
186+
if url.QueryEscape(key) != key {
187+
t.Errorf("should error for 'fromQuery' keys being unacceptable URL characters. orig=%q", orig)
188+
}
189+
190+
// Colons should only be present in values right at the start (they're invalid characters otherwise).
191+
if len(val) > 0 && val[0] == ':' {
192+
val = val[1:]
193+
}
194+
195+
if url.QueryEscape(val) != val {
196+
t.Errorf("should error for 'fromQuery' values containing unacceptable URL characters. orig=%q", orig)
197+
}
198+
}
157199
}
158200

159201
s := bufio.NewScanner(bytes.NewReader(orig))
@@ -172,11 +214,6 @@ func FuzzParse(f *testing.F) {
172214
continue
173215
}
174216

175-
if len(fields) > 3 {
176-
t.Errorf("should error with more than 3 fields. orig=%q", orig)
177-
continue
178-
}
179-
180217
if len(fields) > 0 && !strings.HasPrefix(fields[0], "/") {
181218
t.Errorf("should error for from path not starting with '/'. orig=%q", orig)
182219
continue
@@ -195,3 +232,246 @@ func FuzzParse(f *testing.F) {
195232
}
196233
})
197234
}
235+
236+
func TestMatchAndExpandPlaceholders(t *testing.T) {
237+
testcases := []struct {
238+
name string
239+
rule *Rule
240+
inPath string
241+
inParams string
242+
success bool
243+
expectedTo string
244+
}{
245+
{
246+
name: "No expansion",
247+
rule: &Rule{
248+
From: "/from",
249+
To: "/to",
250+
},
251+
inPath: "/from",
252+
inParams: "",
253+
success: true,
254+
expectedTo: "/to",
255+
},
256+
{
257+
name: "No expansion, but trailing slash",
258+
rule: &Rule{
259+
From: "/from/",
260+
To: "/to",
261+
},
262+
inPath: "/from",
263+
inParams: "",
264+
success: true,
265+
expectedTo: "/to",
266+
},
267+
{
268+
name: "Splat matching",
269+
rule: &Rule{
270+
From: "/*",
271+
To: "/to",
272+
},
273+
inPath: "/from",
274+
inParams: "",
275+
success: true,
276+
expectedTo: "/to",
277+
},
278+
{
279+
name: "Splat substitution",
280+
rule: &Rule{
281+
From: "/*",
282+
To: "/other/:splat",
283+
},
284+
inPath: "/from",
285+
inParams: "",
286+
success: true,
287+
expectedTo: "/other/from",
288+
},
289+
{
290+
name: "Named substitution",
291+
rule: &Rule{
292+
From: "/:thing",
293+
To: "/:thing.html",
294+
},
295+
inPath: "/from",
296+
inParams: "",
297+
success: true,
298+
expectedTo: "/from.html",
299+
},
300+
{
301+
name: "Missing placeholder",
302+
rule: &Rule{
303+
From: "/:this",
304+
To: "/:that.html",
305+
},
306+
inPath: "/from",
307+
inParams: "",
308+
success: false,
309+
},
310+
{
311+
name: "Static query parameter, match",
312+
rule: &Rule{
313+
From: "/from",
314+
FromQuery: map[string]string{
315+
"a": "b",
316+
},
317+
To: "/to",
318+
},
319+
inPath: "/from",
320+
inParams: "a=b",
321+
success: true,
322+
expectedTo: "/to",
323+
},
324+
{
325+
name: "Static query parameter, muli-match first",
326+
rule: &Rule{
327+
From: "/from",
328+
FromQuery: map[string]string{
329+
"a": "b",
330+
},
331+
To: "/to",
332+
},
333+
inPath: "/from",
334+
inParams: "a=b&a=c",
335+
success: true,
336+
expectedTo: "/to",
337+
},
338+
{
339+
name: "Static query parameter, muli-match second",
340+
rule: &Rule{
341+
From: "/from",
342+
FromQuery: map[string]string{
343+
"a": "b",
344+
},
345+
To: "/to",
346+
},
347+
inPath: "/from",
348+
inParams: "a=c&a=b",
349+
success: true,
350+
expectedTo: "/to",
351+
},
352+
{
353+
name: "Static query parameter, no match",
354+
rule: &Rule{
355+
From: "/from",
356+
FromQuery: map[string]string{
357+
"a": "b",
358+
},
359+
To: "/to",
360+
},
361+
inPath: "/from",
362+
inParams: "",
363+
success: false,
364+
},
365+
{
366+
name: "Dynamic query parameter, match",
367+
rule: &Rule{
368+
From: "/from",
369+
FromQuery: map[string]string{
370+
"a": ":a",
371+
},
372+
To: "/to/:a.html",
373+
},
374+
inPath: "/from",
375+
inParams: "a=b",
376+
success: true,
377+
expectedTo: "/to/b.html",
378+
},
379+
{
380+
name: "Dynamic query parameter, multi-match",
381+
rule: &Rule{
382+
From: "/from",
383+
FromQuery: map[string]string{
384+
"a": ":a",
385+
},
386+
To: "/:a.html",
387+
},
388+
inPath: "/from",
389+
inParams: "a=b&a=c",
390+
success: true,
391+
expectedTo: "/b.html",
392+
},
393+
{
394+
name: "Dynamic query parameter, no match",
395+
rule: &Rule{
396+
From: "/from",
397+
FromQuery: map[string]string{
398+
"a": "b",
399+
},
400+
To: "/to",
401+
},
402+
inPath: "/from",
403+
inParams: "",
404+
success: false,
405+
},
406+
{
407+
name: "Repeated placeholder in path",
408+
rule: &Rule{
409+
From: "/:from/:from",
410+
To: "/:from.html",
411+
},
412+
inPath: "/a/b",
413+
inParams: "",
414+
success: true,
415+
expectedTo: "/b.html",
416+
},
417+
{
418+
name: "Repeated placeholder in params",
419+
rule: &Rule{
420+
From: "/from",
421+
FromQuery: map[string]string{
422+
"q": ":val",
423+
"r": ":val",
424+
},
425+
To: "/:val.html",
426+
},
427+
inPath: "/from",
428+
inParams: "q=qq&r=rr",
429+
success: true,
430+
expectedTo: "/qq.html",
431+
},
432+
{
433+
name: "Repeated placeholder in path then params",
434+
rule: &Rule{
435+
From: "/:val",
436+
FromQuery: map[string]string{
437+
"q": ":val",
438+
},
439+
To: "/:val.html",
440+
},
441+
inPath: "/path",
442+
inParams: "q=query",
443+
success: true,
444+
expectedTo: "/path.html",
445+
},
446+
{
447+
name: "Repeated placeholder splat",
448+
rule: &Rule{
449+
From: "/*",
450+
FromQuery: map[string]string{
451+
"q": ":splat",
452+
},
453+
To: "/:splat.html",
454+
},
455+
inPath: "/path",
456+
inParams: "q=query",
457+
success: true,
458+
expectedTo: "/path.html",
459+
},
460+
}
461+
462+
for _, tc := range testcases {
463+
t.Run(tc.name, func(t *testing.T) {
464+
params, err := url.ParseQuery(tc.inParams)
465+
if err != nil {
466+
t.Errorf("Invalid inParams given (%s): %v", tc.inParams, err)
467+
}
468+
469+
ok := tc.rule.MatchAndExpandPlaceholders(tc.inPath, params)
470+
assert.Equal(t, tc.success, ok, "Expected success to be %v, but was %v", tc.success, ok)
471+
472+
if tc.success {
473+
assert.Equal(t, tc.expectedTo, tc.rule.To, "Expected the To property to be changed to %q, but was %q", tc.expectedTo, tc.rule.To)
474+
}
475+
})
476+
}
477+
}

0 commit comments

Comments
 (0)
Please sign in to comment.