diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index e2ea666..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/crawler/crawler.go b/crawler/crawler.go index 83b3cfa..ae7e7b6 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -3,6 +3,7 @@ package crawler import ( "bufio" "bytes" + "fmt" "net" "net/http" "os" @@ -15,8 +16,7 @@ import ( "github.com/gocolly/colly/v2/extensions" "github.com/gocolly/colly/v2/proxy" - "github.com/superiss/colorful" - domainCheck "github.com/superiss/domain" + domainCheck "github.com/twiny/domain" ) var ( @@ -52,7 +52,6 @@ type Spider struct { responses chan []byte // writer *Writer - logger *colorful.Logger err chan *Error } @@ -176,8 +175,7 @@ func NewSpider(setting *Setting) (*Spider, error) { valids: make(chan string, setting.Engine.Worker), }, // - logger: colorful.New(), - err: errs, + err: errs, }, nil } @@ -288,10 +286,10 @@ func (s *Spider) validate(domain string) { } switch valid { case true: - s.logger.Success(valid, " ", rootDomain, "\n") + fmt.Println(valid, " ", rootDomain, "\n") s.writer.valids <- rootDomain case false: - s.logger.Error(valid, " ", rootDomain, "\n") + fmt.Println(valid, " ", rootDomain, "\n") } } } @@ -308,7 +306,7 @@ func (s *Spider) Run() { // open file input, err := os.Open(s.setting.Engine.URLPath) if err != nil { - s.logger.Error("could not open URLs file") + fmt.Println("could not open URLs file") return } @@ -320,7 +318,7 @@ func (s *Spider) Run() { urls <- scanner.Text() } if err := scanner.Err(); err != nil { - s.logger.Error(err) + fmt.Println(err) } // input.Close() diff --git a/go.mod b/go.mod index dc4600e..1e202a3 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ -module github.com/superiss/spidy +module github.com/twiny/spidy -go 1.14 +go 1.16 require ( - github.com/PuerkitoBio/goquery v1.5.1 + github.com/PuerkitoBio/goquery v1.7.1 github.com/gocolly/colly/v2 v2.1.0 - github.com/superiss/colorful v0.0.0-20200608152512-f8e4d206cee2 - github.com/superiss/domain v0.0.0-20200705235949-a23bca9c08b9 - golang.org/x/net v0.0.0-20200707034311-ab3426394381 - gopkg.in/yaml.v2 v2.3.0 + github.com/twiny/domain v0.0.0-20210823211749-c6132412abc4 + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 1ff6028..8171c25 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4= +github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= @@ -57,15 +58,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/superiss/colorful v0.0.0-20200608152512-f8e4d206cee2 h1:UpdrwusMb2TVfZr+NbMTzMFfLKRheehtN4Du3z2f950= -github.com/superiss/colorful v0.0.0-20200608152512-f8e4d206cee2/go.mod h1:urMa276kqyk/+SxGoxRZQVE8DEsggUr1dN4HsMIIj5w= -github.com/superiss/domain v0.0.0-20200705235949-a23bca9c08b9 h1:QGI5U4M+/4FApYr7CBFtXG+D0Njj+7e09FiAhe8NInk= -github.com/superiss/domain v0.0.0-20200705235949-a23bca9c08b9/go.mod h1:edld7/L6ETMQng3b44FqcYDtrw481YEnn1P+HvP5SKc= github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/twiny/domain v0.0.0-20210823211749-c6132412abc4 h1:4H8ekRCSmgmQxfdmARjdOlAZnwu5yEIyrB4bYqlhU5E= +github.com/twiny/domain v0.0.0-20210823211749-c6132412abc4/go.mod h1:Tb1A51T20XWwHbSnf3ZsY3pbXt3flojw8rze+5+ylz0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -80,8 +78,9 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -90,17 +89,24 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -116,8 +122,11 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/input/output.txt b/input/output.txt deleted file mode 100644 index ebf7a2c..0000000 --- a/input/output.txt +++ /dev/null @@ -1,390 +0,0 @@ -http://3dcart.com -http://abeautifulmess.com -http://abetterlemonadestand.com -http://accelerationpartners.com -http://acechapman.com -http://adamenfroy.com -http://adtraction.com -http://advidi.com -http://adzooma.com -http://affilorama.com -http://affise.com -http://ahrefs.com -http://aioma.com -http://alibaba.com -http://alidropship.com -http://alistgroup.com -http://amazon.ca -http://amazon.com -http://amazon.fr -http://amazon.in -http://amazonaws.com -http://anchorloans.com -http://androidauthority.com -http://apacentrepreneur.com -http://apple.com -http://arabclicks.com -http://artefact.com -http://artsandclassy.com -http://avangatenetwork.com -http://awin.com -http://azbigmedia.com -http://baconwrappedbusiness.com -http://bankrate.com -http://bannerflow.com -http://bebiggy.com -http://beeketing.com -http://benbellabooks.com -http://bench.co -http://betterthansure.com -http://bigcommerce.com -http://biggerpockets.com -http://bitdegree.org -http://biteable.com -http://bizjournals.com -http://bluefountainmedia.com -http://boxme.asia -http://branex.ca -http://buffer.com -http://buildcreate.com -http://business2community.com -http://businessinsider.com -http://businessnewsdaily.com -http://businesstown.com -http://byupathway.org -http://cambridge.org -http://capitalistcreations.com -http://capterra.com -http://cashbackworld.com -http://cashoverflow.in -http://cedcommerce.com -http://channelape.com -http://chiefmarketer.com -http://chinabrands.com -http://chron.com -http://civilmediation.org -http://cj.com -http://clickliverpool.com -http://clickworker.com -http://cloudways.com -http://clutch.co -http://cnn.com -http://codeinwp.com -http://computerhope.com -http://consulting.com -http://contentfac.com -http://copyblogger.com -http://coschedule.com -http://coursera.org -http://crazyegg.com -http://curbed.com -http://customerthink.com -http://cxl.com -http://daisycon.com -http://daveramsey.com -http://dealsanddollarsclub.com -http://diggitymarketing.com -http://digitalagencynetwork.com -http://digitalbrandinginstitute.com -http://digitalinformationworld.com -http://digitalmarketer.com -http://digitalmarketinginstitute.com -http://digitalnomadmom.com -http://digitalnomadsoul.com -http://digitaltrafficace.com -http://digitalvidya.com -http://doersempire.com -http://dollarsprout.com -http://dreamhost.com -http://dropified.com -http://dropizi.fr -http://dropshiplifestyle.com -http://dsmtool.com -http://due.com -http://ebizfacts.com -http://ecomdash.com -http://ecommerce-platforms.com -http://ecommercebuff.com -http://ecommercefuel.com -http://ecommerceguide.com -http://ecommercenews.eu -http://ecommercetimes.com -http://econsultancy.com -http://edx.org -http://emergeapp.net -http://empireflippers.com -http://engaiodigital.com -http://entrepreneur.com -http://entrepreneurshipinabox.com -http://eofire.com -http://europeanbusinessreview.com -http://eventige.com -http://evolvecreativegroup.com -http://exchangemarketplace.com -http://expandcart.com -http://ezoic.com -http://facebook.com -http://falcon.io -http://farmers.com -http://fbi.gov -http://finder.com -http://fitsmallbusiness.com -http://fiverr.com -http://flippa.com -http://flipperforce.com -http://flippingcincy.com -http://flippingincome.com -http://forbes.com -http://forextime.com -http://formidableforms.com -http://fortunebuilders.com -http://foxbusiness.com -http://fundera.com -http://futurelearn.com -http://getresponse.com -http://getresponsepages.com -http://globalreach.com -http://godaddy.com -http://goflipsy.com -http://goodfinancialcents.com -http://grantgloballogistics.com -http://gritdaily.com -http://growthgurus.com -http://hacktheentrepreneur.com -http://hbr.org -http://heraldtribune.com -http://hey-xian.com -http://hometoheather.com -http://homevestorsfranchise.com -http://hootsuite.com -http://hostinger.com -http://housebeautiful.com -http://houseofbanks.com -http://howtostartanllc.com -http://hrblock.com -http://hubspot.com -http://hustlr.com -http://i-scoop.eu -http://inc.com -http://incomediary.com -http://indiatimes.com -http://inflowinventory.com -http://influencermarketinghub.com -http://influencive.com -http://informaticsinc.com -http://instagram.com -http://integratedmarketingassociation.org -http://internationalliving.com -http://internetworldstats.com -http://inventorysource.com -http://investedwallet.com -http://investfourmore.com -http://investopedia.com -http://invoice.ng -http://iprhelpdesk.eu -http://jbmediagroupllc.com -http://jeffbullas.com -http://jivochat.com -http://johanneslarsson.com -http://joinhiving.com -http://journalistontherun.com -http://jumpseller.com -http://junglescout.com -http://justlearn.com -http://kabbage.com -http://later.com -http://leelinesourcing.com -http://lendinghome.com -http://letstalkaboutmoney.com -http://lifehack.org -http://lilachbullock.com -http://linguee.com -http://linguee.fr -http://linkedin.com -http://links2revenue.com -http://linnworks.com -http://localfirstbank.com -http://locationrebel.com -http://logiwa.com -http://loomsolar.com -http://lovemoney.com -http://lufthansa.com -http://lyfemarketing.com -http://lynda.com -http://magenticians.com -http://mailchimp.com -http://mainstreethost.com -http://makeawebsitehub.com -http://marketbusinessnews.com -http://marketingdive.com -http://marketinginsidergroup.com -http://marketingland.com -http://marketingterms.com -http://marketo.com -http://marketplace.org -http://marketsplash.com -http://martechadvisor.com -http://mashable.com -http://mashvisor.com -http://medium.com -http://merchantmaverick.com -http://michelfortin.com -http://mikegingerich.com -http://mint.com -http://missaffiliate.com -http://mobidea.com -http://modalyst.co -http://moneycrashers.com -http://moneysavingexpert.com -http://moneytalksnews.com -http://moneyunder30.com -http://moz.com -http://multichannelmerchant.com -http://mygiftcardsplus.com -http://mypoints.com -http://mywifequitherjob.com -http://namecheap.com -http://nashmimarketing.com -http://neilpatel.com -http://nerdwallet.com -http://nevilleamehra.com -http://newagainhouses.com -http://ninjaacquisitions.com -http://oberlo.com -http://opinionoutpost.com -http://optinmonster.com -http://orlandosentinel.com -http://outbrain.com -http://outgrow.co -http://ownr.co -http://oxidian.hr -http://paldesk.com -http://parade.com -http://payoneer.com -http://photoslurp.com -http://pinterest.com -http://popwebdesign.net -http://primaerp.com -http://printify.com -http://propertygeek.net -http://propy.com -http://publicity.com -http://quicksprout.com -http://quora.com -http://realestateexpress.com -http://realestatesalesllc.com -http://rebelinteractivegroup.com -http://reddit.com -http://referralcandy.com -http://rehabfinancial.com -http://rehabs2riches.com -http://reliablesoft.net -http://repricerexpress.com -http://rockcontent.com -http://rowleylegal.com -http://royallegalsolutions.com -http://ryanwashere.com -http://ryrob.com -http://ryte.com -http://salehoo.com -http://salesandorders.com -http://saltinourhair.com -http://savethestudent.org -http://savvynewcanadians.com -http://sba.gov -http://sciencedirect.com -http://scienceofcreatingwealth.com -http://scottalanturner.com -http://searchenginejournal.com -http://searchengineland.com -http://seekcapital.com -http://sendpulse.com -http://seo.com -http://shawacademy.com -http://shopathome.com -http://shopify.com -http://shopify.hk -http://shopify.no -http://shopifycompass.com -http://shoutmeloud.com -http://shutterstock.com -http://sidehustlehq.com -http://similarweb.com -http://smallbiztrends.com -http://smartaboutmoney.org -http://smartblogger.com -http://smartinsights.com -http://smartpassiveincome.com -http://smemark.com -http://snapagency.com -http://snehiltalks.com -http://socialbakers.com -http://socialmediacollege.com -http://socialmediaexaminer.com -http://socialmediamarketing.com -http://socialmediatoday.com -http://socialpilot.co -http://soffront.com -http://sourcesofinsight.com -http://sourcingjournal.com -http://spikeinteractive.net -http://spocket.co -http://sprinkdigital.com -http://sproutsocial.com -http://startbusinessjourney.com -http://suntecmarketing.com -http://superiorschoolnc.com -http://swagbucks.com -http://talkbusiness.net -http://tapairportugal.com -http://target.com -http://taxsolutionscanada.com -http://techcrunch.com -http://techtarget.com -http://thebalancesmb.com -http://theflippingninja.com -http://themanifest.com -http://themodestwallet.com -http://thenextscoop.com -http://thepma.org -http://therealdeal.com -http://theselfemployed.com -http://thesimpledollar.com -http://thriveagency.com -http://toolbox.com -http://tradetracker.com -http://travelpayouts.com -http://trustradius.com -http://tutsplus.com -http://tweakyourbiz.com -http://udemy.com -http://uplandsoftware.com -http://usatoday.com -http://usertesting.com -http://usnews.com -http://varstreetinc.com -http://veneratedigital.com -http://ventureharbour.com -http://visiture.com -http://voluum.com -http://walmart.com -http://washakieorganic.com -http://washingtonpost.com -http://webfx.com -http://websitebuilderexpert.com -http://whnt.com -http://wikihow.com -http://wikipedia.org -http://wiktionary.org -http://wix.com -http://woocommerce.com -http://wordpressprotips.com -http://wordstream.com -http://wsiworld.com -http://wsj.com -http://wtop.com -http://x-cart.com -http://yahoo.com -http://yakkyofy.com -http://youpreneur.com -http://zoho.com \ No newline at end of file diff --git a/input/test.txt b/input/test.txt deleted file mode 100644 index 28feab1..0000000 --- a/input/test.txt +++ /dev/null @@ -1 +0,0 @@ -https://clickgum.com \ No newline at end of file diff --git a/main.go b/main.go index 50ae3cb..90a134c 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/superiss/spidy/crawler" + "github.com/twiny/spidy/crawler" ) const msg = `