-
Notifications
You must be signed in to change notification settings - Fork 471
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
stock
committed
Jan 13, 2015
1 parent
7b30c2b
commit ffedfe4
Showing
10 changed files
with
353 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,79 @@ | ||
| // Package request implements request entity contains url and other relevant informaion. | ||
| package request | ||
|
|
||
| import ( | ||
| "net/http" | ||
| ) | ||
|
|
||
| // Request represents object waiting for being crawled. | ||
| type Request struct { | ||
| url string | ||
| url string | ||
|
|
||
| // Responce type: html json jsonp text | ||
| respType string | ||
|
|
||
| // GET POST | ||
| method string | ||
|
|
||
| // POST data | ||
| postdata string | ||
|
|
||
| // name for marking url and distinguish different urls in PageProcesser and Pipeline | ||
| urltag string | ||
|
|
||
| // http header | ||
| header http.Header | ||
|
|
||
| // http cookies | ||
| cookies []*http.Cookie | ||
|
|
||
| // Redirect function for downloader used in http.Client | ||
| // If CheckRedirect returns an error, the Client's Get | ||
| // method returns both the previous Response. | ||
| // If CheckRedirect returns error.New("normal"), the error process after client.Do will ignore the error. | ||
| checkRedirect func(req *http.Request, via []*http.Request) error | ||
| } | ||
|
|
||
| // NewRequest returns initialized Request object. | ||
| // The respType is "json" or "html" | ||
| func NewRequest(url string, respType string) *Request { | ||
| return &Request{url, respType} | ||
| // The respType is json, jsonp, html, text | ||
| /* | ||
| func NewRequestSimple(url string, respType string, urltag string) *Request { | ||
| return &Request{url:url, respType:respType} | ||
| } | ||
| */ | ||
|
|
||
| func NewRequest(url string, respType string, urltag string, method string, postdata string, header http.Header, cookies []*http.Cookie, checkRedirect func(req *http.Request, via []*http.Request) error) *Request { | ||
| return &Request{url, respType, method, postdata, urltag, header, cookies, checkRedirect} | ||
| } | ||
|
|
||
| func (this *Request) GetUrl() string { | ||
| return this.url | ||
| } | ||
|
|
||
| func (this *Request) GetUrlTag() string { | ||
| return this.urltag | ||
| } | ||
|
|
||
| func (this *Request) GetMethod() string { | ||
| return this.method | ||
| } | ||
|
|
||
| func (this *Request) GetPostdata() string { | ||
| return this.postdata | ||
| } | ||
|
|
||
| func (this *Request) GetHeader() http.Header { | ||
| return this.header | ||
| } | ||
|
|
||
| func (this *Request) GetCookies() []*http.Cookie { | ||
| return this.cookies | ||
| } | ||
|
|
||
| func (this *Request) GetResponceType() string { | ||
| return this.respType | ||
| } | ||
|
|
||
| func (this *Request) GetRedirectFunc() func(req *http.Request, via []*http.Request) error { | ||
| return this.checkRedirect | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Copyright 2014 Hu Cong. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| // The package is useless | ||
| package scheduler | ||
|
|
||
| import ( | ||
| "github.com/hu17889/go_spider/core/common/request" | ||
| ) | ||
|
|
||
| type SimpleScheduler struct { | ||
| queue chan *request.Request | ||
| } | ||
|
|
||
| func NewSimpleScheduler() *SimpleScheduler { | ||
| ch := make(chan *request.Request, 1024) | ||
| return &SimpleScheduler{ch} | ||
| } | ||
|
|
||
| func (this *SimpleScheduler) Push(requ *request.Request) { | ||
| this.queue <- requ | ||
| } | ||
|
|
||
| func (this *SimpleScheduler) Poll() *request.Request { | ||
| if len(this.queue) == 0 { | ||
| return nil | ||
| } else { | ||
| return <-this.queue | ||
| } | ||
| } | ||
|
|
||
| func (this *SimpleScheduler) Count() int { | ||
| return len(this.queue) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.