yakumo.izuru/yukari
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[enh] disallow crawling bots

Browse Source 
git-svn-id: file:///srv/svn/repo/yukari/trunk@10 f3bd38d9-da89-464d-a02a-eb04e43141b5
This commit is contained in:
asciimoo 2016-10-25 14:55:06 +00:00
parent 5dee40ae5c
commit 594bc85db6
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
morty.go
View File

@ -123,6 +123,11 @@ input[type=checkbox]#mortytoggle:checked ~ div { display: none; }
`
func (p *Proxy) RequestHandler(ctx *fasthttp.RequestCtx) {
if appRequestHandler(ctx) {
return
}
requestHash := popRequestParam(ctx, []byte("mortyhash"))
requestURI := popRequestParam(ctx, []byte("mortyurl"))
@ -225,6 +230,15 @@ func (p *Proxy) RequestHandler(ctx *fasthttp.RequestCtx) {
}
}
func appRequestHandler(ctx *fasthttp.RequestCtx) bool {
if bytes.Equal(ctx.Path(), []byte("/robots.txt")) {
ctx.SetContentType("text/plain")
ctx.Write([]byte("User-Agent: *\nDisallow: /\n"))
return true
}
return false
}
func popRequestParam(ctx *fasthttp.RequestCtx, paramName []byte) []byte {
param := ctx.QueryArgs().PeekBytes(paramName)