From 04f4b72182eaf5533275433be6933d15932af480 Mon Sep 17 00:00:00 2001 From: Eugene Dementiev Date: Wed, 28 Oct 2015 22:41:57 +0300 Subject: [PATCH] [plugin] logstreamer --- Godeps/Godeps.json | 44 +- .../src/github.com/howeyc/fsnotify/.gitignore | 5 + .../src/github.com/howeyc/fsnotify/AUTHORS | 28 + .../github.com/howeyc/fsnotify/CHANGELOG.md | 160 +++ .../howeyc/fsnotify/CONTRIBUTING.md | 7 + .../src/github.com/howeyc/fsnotify/LICENSE | 28 + .../src/github.com/howeyc/fsnotify/README.md | 93 ++ .../howeyc/fsnotify/example_test.go | 34 + .../github.com/howeyc/fsnotify/fsnotify.go | 111 ++ .../howeyc/fsnotify/fsnotify_bsd.go | 496 +++++++ .../howeyc/fsnotify/fsnotify_linux.go | 304 +++++ .../howeyc/fsnotify/fsnotify_open_bsd.go | 11 + .../howeyc/fsnotify/fsnotify_open_darwin.go | 11 + .../howeyc/fsnotify/fsnotify_symlink_test.go | 74 ++ .../howeyc/fsnotify/fsnotify_test.go | 1010 +++++++++++++++ .../howeyc/fsnotify/fsnotify_windows.go | 598 +++++++++ .../src/github.com/hpcloud/tail/.gitignore | 3 + .../src/github.com/hpcloud/tail/.travis.yml | 14 + .../src/github.com/hpcloud/tail/CHANGES.md | 57 + .../src/github.com/hpcloud/tail/Dockerfile | 19 + .../src/github.com/hpcloud/tail/LICENSE.txt | 23 + .../src/github.com/hpcloud/tail/Makefile | 11 + .../src/github.com/hpcloud/tail/README.md | 27 + .../hpcloud/tail/cmd/gotail/.gitignore | 1 + .../hpcloud/tail/cmd/gotail/Makefile | 4 + .../hpcloud/tail/cmd/gotail/gotail.go | 64 + .../src/github.com/hpcloud/tail/deps.json | 14 + .../hpcloud/tail/ratelimiter/Licence | 7 + .../hpcloud/tail/ratelimiter/leakybucket.go | 97 ++ .../tail/ratelimiter/leakybucket_test.go | 73 ++ .../hpcloud/tail/ratelimiter/memory.go | 58 + .../hpcloud/tail/ratelimiter/storage.go | 6 + .../src/github.com/hpcloud/tail/tail.go | 406 ++++++ .../src/github.com/hpcloud/tail/tail_posix.go | 11 + .../src/github.com/hpcloud/tail/tail_test.go | 438 +++++++ .../github.com/hpcloud/tail/tail_windows.go | 12 + .../src/github.com/hpcloud/tail/util/util.go | 47 + .../hpcloud/tail/watch/filechanges.go | 42 + .../github.com/hpcloud/tail/watch/inotify.go | 123 ++ .../hpcloud/tail/watch/inotify_tracker.go | 169 +++ .../github.com/hpcloud/tail/watch/polling.go | 118 ++ .../github.com/hpcloud/tail/watch/watch.go | 19 + .../hpcloud/tail/winfile/winfile.go | 92 ++ .../src/gopkg.in/fsnotify.v1/.gitignore | 6 + .../src/gopkg.in/fsnotify.v1/.travis.yml | 20 + .../src/gopkg.in/fsnotify.v1/AUTHORS | 34 + .../src/gopkg.in/fsnotify.v1/CHANGELOG.md | 267 ++++ .../src/gopkg.in/fsnotify.v1/CONTRIBUTING.md | 77 ++ .../src/gopkg.in/fsnotify.v1/LICENSE | 28 + .../src/gopkg.in/fsnotify.v1/README.md | 64 + .../src/gopkg.in/fsnotify.v1/example_test.go | 42 + .../src/gopkg.in/fsnotify.v1/fsnotify.go | 62 + .../src/gopkg.in/fsnotify.v1/inotify.go | 306 +++++ .../gopkg.in/fsnotify.v1/inotify_poller.go | 186 +++ .../fsnotify.v1/inotify_poller_test.go | 228 ++++ .../src/gopkg.in/fsnotify.v1/inotify_test.go | 292 +++++ .../gopkg.in/fsnotify.v1/integration_test.go | 1135 +++++++++++++++++ .../src/gopkg.in/fsnotify.v1/kqueue.go | 468 +++++++ .../src/gopkg.in/fsnotify.v1/open_mode_bsd.go | 11 + .../gopkg.in/fsnotify.v1/open_mode_darwin.go | 12 + .../src/gopkg.in/fsnotify.v1/windows.go | 561 ++++++++ .../_workspace/src/gopkg.in/tomb.v1/LICENSE | 29 + .../_workspace/src/gopkg.in/tomb.v1/README.md | 4 + .../_workspace/src/gopkg.in/tomb.v1/tomb.go | 176 +++ .../src/gopkg.in/tomb.v1/tomb_test.go | 114 ++ plugins/all/all.go | 1 + plugins/logstreamer/logstream/files.go | 203 +++ plugins/logstreamer/logstream/metric.go | 15 + plugins/logstreamer/logstream/rule.go | 82 ++ plugins/logstreamer/logstreamer.go | 129 ++ 70 files changed, 9507 insertions(+), 14 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/.gitignore create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/AUTHORS create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/CHANGELOG.md create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/CONTRIBUTING.md create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/LICENSE create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/README.md create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/example_test.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_linux.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_darwin.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_symlink_test.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_test.go create mode 100644 Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_windows.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/.gitignore create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/CHANGES.md create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/Dockerfile create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/LICENSE.txt create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/Makefile create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/README.md create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/.gitignore create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/Makefile create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/gotail.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/deps.json create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/Licence create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket_test.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/memory.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/storage.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/tail.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/tail_posix.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/tail_test.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/tail_windows.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/util/util.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/watch/filechanges.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify_tracker.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/watch/polling.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/watch/watch.go create mode 100644 Godeps/_workspace/src/github.com/hpcloud/tail/winfile/winfile.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/.gitignore create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/CONTRIBUTING.md create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/LICENSE create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/example_test.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/fsnotify.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller_test.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_bsd.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_darwin.go create mode 100644 Godeps/_workspace/src/gopkg.in/fsnotify.v1/windows.go create mode 100644 Godeps/_workspace/src/gopkg.in/tomb.v1/LICENSE create mode 100644 Godeps/_workspace/src/gopkg.in/tomb.v1/README.md create mode 100644 Godeps/_workspace/src/gopkg.in/tomb.v1/tomb.go create mode 100644 Godeps/_workspace/src/gopkg.in/tomb.v1/tomb_test.go create mode 100644 plugins/logstreamer/logstream/files.go create mode 100644 plugins/logstreamer/logstream/metric.go create mode 100644 plugins/logstreamer/logstream/rule.go create mode 100644 plugins/logstreamer/logstreamer.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 5c38b2681382b..0d35795066372 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,9 +1,6 @@ { "ImportPath": "github.com/influxdb/telegraf", - "GoVersion": "go1.5.1", - "Packages": [ - "./..." - ], + "GoVersion": "go1.5", "Deps": [ { "ImportPath": "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git", @@ -100,6 +97,16 @@ "ImportPath": "github.com/hashicorp/raft-boltdb", "Rev": "d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee" }, + { + "ImportPath": "github.com/howeyc/fsnotify", + "Comment": "v0.9.0-16-gf0c08ee", + "Rev": "f0c08ee9c60704c1879025f2ae0ff3e000082c13" + }, + { + "ImportPath": "github.com/hpcloud/tail", + "Comment": "v0-29-g0f28969", + "Rev": "0f289693ec9e9b6a605d7ad1262a8d8c4338ed77" + }, { "ImportPath": "github.com/influxdb/influxdb", "Comment": "v0.9.4-rc1-703-g956efae", @@ -114,6 +121,11 @@ "ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil", "Rev": "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a" }, + { + "ImportPath": "github.com/mreiferson/go-snappystream", + "Comment": "v0.2.3", + "Rev": "028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504" + }, { "ImportPath": "github.com/naoina/go-stringutil", "Rev": "360db0db4b01d34e12a2ec042c09e7d37fece761" @@ -122,6 +134,11 @@ "ImportPath": "github.com/naoina/toml", "Rev": "5811abcabb29d6af0fdf060f96d328962bd3cd5e" }, + { + "ImportPath": "github.com/nsqio/go-nsq", + "Comment": "v1.0.5-6-g2118015", + "Rev": "2118015c120962edc5d03325c680daf3163a8b5f" + }, { "ImportPath": "github.com/pborman/uuid", "Rev": "ca53cad383cad2479bbba7f7a1a05797ec1386e4" @@ -252,24 +269,23 @@ "ImportPath": "gopkg.in/fatih/pool.v2", "Rev": "cba550ebf9bce999a02e963296d4bc7a486cb715" }, + { + "ImportPath": "gopkg.in/fsnotify.v1", + "Comment": "v1.2.1", + "Rev": "7be54206639f256967dd82fa767397ba5f8f48f5" + }, { "ImportPath": "gopkg.in/mgo.v2", "Comment": "r2015.06.03-3-g3569c88", "Rev": "3569c88678d88179dcbd68d02ab081cbca3cd4d0" }, { - "ImportPath": "gopkg.in/yaml.v2", - "Rev": "7ad95dd0798a40da1ccdff6dff35fd177b5edf40" - }, - { - "ImportPath": "github.com/mreiferson/go-snappystream", - "Comment": "v0.2.3", - "Rev": "028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504" + "ImportPath": "gopkg.in/tomb.v1", + "Rev": "dd632973f1e7218eb1089048e0798ec9ae7dceb8" }, { - "ImportPath": "github.com/nsqio/go-nsq", - "Comment": "v1.0.5-6-g2118015", - "Rev": "2118015c120962edc5d03325c680daf3163a8b5f" + "ImportPath": "gopkg.in/yaml.v2", + "Rev": "7ad95dd0798a40da1ccdff6dff35fd177b5edf40" } ] } diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/.gitignore b/Godeps/_workspace/src/github.com/howeyc/fsnotify/.gitignore new file mode 100644 index 0000000000000..e4706a9e91c85 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/.gitignore @@ -0,0 +1,5 @@ +# Setup a Global .gitignore for OS and editor generated files: +# https://help.github.com/articles/ignoring-files +# git config --global core.excludesfile ~/.gitignore_global + +.vagrant diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/AUTHORS b/Godeps/_workspace/src/github.com/howeyc/fsnotify/AUTHORS new file mode 100644 index 0000000000000..e52b72f83a187 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/AUTHORS @@ -0,0 +1,28 @@ +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# You can update this list using the following command: +# +# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' + +# Please keep the list sorted. + +Adrien Bustany +Caleb Spare +Case Nelson +Chris Howey +Christoffer Buchholz +Dave Cheney +Francisco Souza +John C Barstow +Kelvin Fo +Nathan Youngman +Paul Hammond +Pursuit92 +Rob Figueiredo +Travis Cline +Tudor Golubenco +bronze1man +debrando +henrikedwards diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/CHANGELOG.md b/Godeps/_workspace/src/github.com/howeyc/fsnotify/CHANGELOG.md new file mode 100644 index 0000000000000..761686aa9539f --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/CHANGELOG.md @@ -0,0 +1,160 @@ +# Changelog + +## v0.9.0 / 2014-01-17 + +* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) +* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) +* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. + +## v0.8.12 / 2013-11-13 + +* [API] Remove FD_SET and friends from Linux adapter + +## v0.8.11 / 2013-11-02 + +* [Doc] Add Changelog [#72][] (thanks @nathany) +* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond) + +## v0.8.10 / 2013-10-19 + +* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) +* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) +* [Doc] specify OS-specific limits in README (thanks @debrando) + +## v0.8.9 / 2013-09-08 + +* [Doc] Contributing (thanks @nathany) +* [Doc] update package path in example code [#63][] (thanks @paulhammond) +* [Doc] GoCI badge in README (Linux only) [#60][] +* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) + +## v0.8.8 / 2013-06-17 + +* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) + +## v0.8.7 / 2013-06-03 + +* [API] Make syscall flags internal +* [Fix] inotify: ignore event changes +* [Fix] race in symlink test [#45][] (reported by @srid) +* [Fix] tests on Windows +* lower case error messages + +## v0.8.6 / 2013-05-23 + +* kqueue: Use EVT_ONLY flag on Darwin +* [Doc] Update README with full example + +## v0.8.5 / 2013-05-09 + +* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) + +## v0.8.4 / 2013-04-07 + +* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) + +## v0.8.3 / 2013-03-13 + +* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) +* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) + +## v0.8.2 / 2013-02-07 + +* [Doc] add Authors +* [Fix] fix data races for map access [#29][] (thanks @fsouza) + +## v0.8.1 / 2013-01-09 + +* [Fix] Windows path separators +* [Doc] BSD License + +## v0.8.0 / 2012-11-09 + +* kqueue: directory watching improvements (thanks @vmirage) +* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) +* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) + +## v0.7.4 / 2012-10-09 + +* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) +* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) +* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) +* [Fix] kqueue: modify after recreation of file + +## v0.7.3 / 2012-09-27 + +* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) +* [Fix] kqueue: no longer get duplicate CREATE events + +## v0.7.2 / 2012-09-01 + +* kqueue: events for created directories + +## v0.7.1 / 2012-07-14 + +* [Fix] for renaming files + +## v0.7.0 / 2012-07-02 + +* [Feature] FSNotify flags +* [Fix] inotify: Added file name back to event path + +## v0.6.0 / 2012-06-06 + +* kqueue: watch files after directory created (thanks @tmc) + +## v0.5.1 / 2012-05-22 + +* [Fix] inotify: remove all watches before Close() + +## v0.5.0 / 2012-05-03 + +* [API] kqueue: return errors during watch instead of sending over channel +* kqueue: match symlink behavior on Linux +* inotify: add `DELETE_SELF` (requested by @taralx) +* [Fix] kqueue: handle EINTR (reported by @robfig) +* [Doc] Godoc example [#1][] (thanks @davecheney) + +## v0.4.0 / 2012-03-30 + +* Go 1 released: build with go tool +* [Feature] Windows support using winfsnotify +* Windows does not have attribute change notifications +* Roll attribute notifications into IsModify + +## v0.3.0 / 2012-02-19 + +* kqueue: add files when watch directory + +## v0.2.0 / 2011-12-30 + +* update to latest Go weekly code + +## v0.1.0 / 2011-10-19 + +* kqueue: add watch on file creation to match inotify +* kqueue: create file event +* inotify: ignore `IN_IGNORED` events +* event String() +* linux: common FileEvent functions +* initial commit + +[#79]: https://github.com/howeyc/fsnotify/pull/79 +[#77]: https://github.com/howeyc/fsnotify/pull/77 +[#72]: https://github.com/howeyc/fsnotify/issues/72 +[#71]: https://github.com/howeyc/fsnotify/issues/71 +[#70]: https://github.com/howeyc/fsnotify/issues/70 +[#63]: https://github.com/howeyc/fsnotify/issues/63 +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#60]: https://github.com/howeyc/fsnotify/issues/60 +[#59]: https://github.com/howeyc/fsnotify/issues/59 +[#49]: https://github.com/howeyc/fsnotify/issues/49 +[#45]: https://github.com/howeyc/fsnotify/issues/45 +[#40]: https://github.com/howeyc/fsnotify/issues/40 +[#36]: https://github.com/howeyc/fsnotify/issues/36 +[#33]: https://github.com/howeyc/fsnotify/issues/33 +[#29]: https://github.com/howeyc/fsnotify/issues/29 +[#25]: https://github.com/howeyc/fsnotify/issues/25 +[#24]: https://github.com/howeyc/fsnotify/issues/24 +[#21]: https://github.com/howeyc/fsnotify/issues/21 +[#1]: https://github.com/howeyc/fsnotify/issues/1 diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/CONTRIBUTING.md b/Godeps/_workspace/src/github.com/howeyc/fsnotify/CONTRIBUTING.md new file mode 100644 index 0000000000000..b2025d72c3957 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/CONTRIBUTING.md @@ -0,0 +1,7 @@ +# Contributing + +## Moving Notice + +There is a fork being actively developed with a new API in preparation for the Go Standard Library: +[github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify) + diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/LICENSE b/Godeps/_workspace/src/github.com/howeyc/fsnotify/LICENSE new file mode 100644 index 0000000000000..f21e54080090f --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/README.md b/Godeps/_workspace/src/github.com/howeyc/fsnotify/README.md new file mode 100644 index 0000000000000..7fdaf7c995e7d --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/README.md @@ -0,0 +1,93 @@ +# File system notifications for Go + +[![GoDoc](https://godoc.org/github.com/howeyc/fsnotify?status.png)](http://godoc.org/github.com/howeyc/fsnotify) + +Cross platform: Windows, Linux, BSD and OS X. + +## Moving Notice + +There is a fork being actively developed with a new API in preparation for the Go Standard Library: +[github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify) + +## Example: + +```go +package main + +import ( + "log" + + "github.com/howeyc/fsnotify" +) + +func main() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + done := make(chan bool) + + // Process events + go func() { + for { + select { + case ev := <-watcher.Event: + log.Println("event:", ev) + case err := <-watcher.Error: + log.Println("error:", err) + } + } + }() + + err = watcher.Watch("testDir") + if err != nil { + log.Fatal(err) + } + + // Hang so program doesn't exit + <-done + + /* ... do stuff ... */ + watcher.Close() +} +``` + +For each event: +* Name +* IsCreate() +* IsDelete() +* IsModify() +* IsRename() + +## FAQ + +**When a file is moved to another directory is it still being watched?** + +No (it shouldn't be, unless you are watching where it was moved to). + +**When I watch a directory, are all subdirectories watched as well?** + +No, you must add watches for any directory you want to watch (a recursive watcher is in the works [#56][]). + +**Do I have to watch the Error and Event channels in a separate goroutine?** + +As of now, yes. Looking into making this single-thread friendly (see [#7][]) + +**Why am I receiving multiple events for the same file on OS X?** + +Spotlight indexing on OS X can result in multiple events (see [#62][]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#54][]). + +**How many files can be watched at once?** + +There are OS-specific limits as to how many watches can be created: +* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, +reaching this limit results in a "no space left on device" error. +* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. + + +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#56]: https://github.com/howeyc/fsnotify/issues/56 +[#54]: https://github.com/howeyc/fsnotify/issues/54 +[#7]: https://github.com/howeyc/fsnotify/issues/7 + diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/example_test.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/example_test.go new file mode 100644 index 0000000000000..d3130e2226421 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/example_test.go @@ -0,0 +1,34 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fsnotify_test + +import ( + "log" + + "github.com/howeyc/fsnotify" +) + +func ExampleNewWatcher() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + go func() { + for { + select { + case ev := <-watcher.Event: + log.Println("event:", ev) + case err := <-watcher.Error: + log.Println("error:", err) + } + } + }() + + err = watcher.Watch("/tmp/foo") + if err != nil { + log.Fatal(err) + } +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify.go new file mode 100644 index 0000000000000..9a48d847d3a5f --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify.go @@ -0,0 +1,111 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fsnotify implements file system notification. +package fsnotify + +import "fmt" + +const ( + FSN_CREATE = 1 + FSN_MODIFY = 2 + FSN_DELETE = 4 + FSN_RENAME = 8 + + FSN_ALL = FSN_MODIFY | FSN_DELETE | FSN_RENAME | FSN_CREATE +) + +// Purge events from interal chan to external chan if passes filter +func (w *Watcher) purgeEvents() { + for ev := range w.internalEvent { + sendEvent := false + w.fsnmut.Lock() + fsnFlags := w.fsnFlags[ev.Name] + w.fsnmut.Unlock() + + if (fsnFlags&FSN_CREATE == FSN_CREATE) && ev.IsCreate() { + sendEvent = true + } + + if (fsnFlags&FSN_MODIFY == FSN_MODIFY) && ev.IsModify() { + sendEvent = true + } + + if (fsnFlags&FSN_DELETE == FSN_DELETE) && ev.IsDelete() { + sendEvent = true + } + + if (fsnFlags&FSN_RENAME == FSN_RENAME) && ev.IsRename() { + sendEvent = true + } + + if sendEvent { + w.Event <- ev + } + + // If there's no file, then no more events for user + // BSD must keep watch for internal use (watches DELETEs to keep track + // what files exist for create events) + if ev.IsDelete() { + w.fsnmut.Lock() + delete(w.fsnFlags, ev.Name) + w.fsnmut.Unlock() + } + } + + close(w.Event) +} + +// Watch a given file path +func (w *Watcher) Watch(path string) error { + return w.WatchFlags(path, FSN_ALL) +} + +// Watch a given file path for a particular set of notifications (FSN_MODIFY etc.) +func (w *Watcher) WatchFlags(path string, flags uint32) error { + w.fsnmut.Lock() + w.fsnFlags[path] = flags + w.fsnmut.Unlock() + return w.watch(path) +} + +// Remove a watch on a file +func (w *Watcher) RemoveWatch(path string) error { + w.fsnmut.Lock() + delete(w.fsnFlags, path) + w.fsnmut.Unlock() + return w.removeWatch(path) +} + +// String formats the event e in the form +// "filename: DELETE|MODIFY|..." +func (e *FileEvent) String() string { + var events string = "" + + if e.IsCreate() { + events += "|" + "CREATE" + } + + if e.IsDelete() { + events += "|" + "DELETE" + } + + if e.IsModify() { + events += "|" + "MODIFY" + } + + if e.IsRename() { + events += "|" + "RENAME" + } + + if e.IsAttrib() { + events += "|" + "ATTRIB" + } + + if len(events) > 0 { + events = events[1:] + } + + return fmt.Sprintf("%q: %s", e.Name, events) +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go new file mode 100644 index 0000000000000..40c84768bc34c --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_bsd.go @@ -0,0 +1,496 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly darwin + +package fsnotify + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + "syscall" +) + +const ( + // Flags (from ) + sys_NOTE_DELETE = 0x0001 /* vnode was removed */ + sys_NOTE_WRITE = 0x0002 /* data contents changed */ + sys_NOTE_EXTEND = 0x0004 /* size increased */ + sys_NOTE_ATTRIB = 0x0008 /* attributes changed */ + sys_NOTE_LINK = 0x0010 /* link count changed */ + sys_NOTE_RENAME = 0x0020 /* vnode was renamed */ + sys_NOTE_REVOKE = 0x0040 /* vnode access was revoked */ + + // Watch all events + sys_NOTE_ALLEVENTS = sys_NOTE_DELETE | sys_NOTE_WRITE | sys_NOTE_ATTRIB | sys_NOTE_RENAME + + // Block for 100 ms on each call to kevent + keventWaitTime = 100e6 +) + +type FileEvent struct { + mask uint32 // Mask of events + Name string // File name (optional) + create bool // set by fsnotify package if found new file +} + +// IsCreate reports whether the FileEvent was triggered by a creation +func (e *FileEvent) IsCreate() bool { return e.create } + +// IsDelete reports whether the FileEvent was triggered by a delete +func (e *FileEvent) IsDelete() bool { return (e.mask & sys_NOTE_DELETE) == sys_NOTE_DELETE } + +// IsModify reports whether the FileEvent was triggered by a file modification +func (e *FileEvent) IsModify() bool { + return ((e.mask&sys_NOTE_WRITE) == sys_NOTE_WRITE || (e.mask&sys_NOTE_ATTRIB) == sys_NOTE_ATTRIB) +} + +// IsRename reports whether the FileEvent was triggered by a change name +func (e *FileEvent) IsRename() bool { return (e.mask & sys_NOTE_RENAME) == sys_NOTE_RENAME } + +// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata. +func (e *FileEvent) IsAttrib() bool { + return (e.mask & sys_NOTE_ATTRIB) == sys_NOTE_ATTRIB +} + +type Watcher struct { + mu sync.Mutex // Mutex for the Watcher itself. + kq int // File descriptor (as returned by the kqueue() syscall) + watches map[string]int // Map of watched file descriptors (key: path) + wmut sync.Mutex // Protects access to watches. + fsnFlags map[string]uint32 // Map of watched files to flags used for filter + fsnmut sync.Mutex // Protects access to fsnFlags. + enFlags map[string]uint32 // Map of watched files to evfilt note flags used in kqueue + enmut sync.Mutex // Protects access to enFlags. + paths map[int]string // Map of watched paths (key: watch descriptor) + finfo map[int]os.FileInfo // Map of file information (isDir, isReg; key: watch descriptor) + pmut sync.Mutex // Protects access to paths and finfo. + fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events) + femut sync.Mutex // Protects access to fileExists. + externalWatches map[string]bool // Map of watches added by user of the library. + ewmut sync.Mutex // Protects access to externalWatches. + Error chan error // Errors are sent on this channel + internalEvent chan *FileEvent // Events are queued on this channel + Event chan *FileEvent // Events are returned on this channel + done chan bool // Channel for sending a "quit message" to the reader goroutine + isClosed bool // Set to true when Close() is first called +} + +// NewWatcher creates and returns a new kevent instance using kqueue(2) +func NewWatcher() (*Watcher, error) { + fd, errno := syscall.Kqueue() + if fd == -1 { + return nil, os.NewSyscallError("kqueue", errno) + } + w := &Watcher{ + kq: fd, + watches: make(map[string]int), + fsnFlags: make(map[string]uint32), + enFlags: make(map[string]uint32), + paths: make(map[int]string), + finfo: make(map[int]os.FileInfo), + fileExists: make(map[string]bool), + externalWatches: make(map[string]bool), + internalEvent: make(chan *FileEvent), + Event: make(chan *FileEvent), + Error: make(chan error), + done: make(chan bool, 1), + } + + go w.readEvents() + go w.purgeEvents() + return w, nil +} + +// Close closes a kevent watcher instance +// It sends a message to the reader goroutine to quit and removes all watches +// associated with the kevent instance +func (w *Watcher) Close() error { + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return nil + } + w.isClosed = true + w.mu.Unlock() + + // Send "quit" message to the reader goroutine + w.done <- true + w.wmut.Lock() + ws := w.watches + w.wmut.Unlock() + for path := range ws { + w.removeWatch(path) + } + + return nil +} + +// AddWatch adds path to the watched file set. +// The flags are interpreted as described in kevent(2). +func (w *Watcher) addWatch(path string, flags uint32) error { + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return errors.New("kevent instance already closed") + } + w.mu.Unlock() + + watchDir := false + + w.wmut.Lock() + watchfd, found := w.watches[path] + w.wmut.Unlock() + if !found { + fi, errstat := os.Lstat(path) + if errstat != nil { + return errstat + } + + // don't watch socket + if fi.Mode()&os.ModeSocket == os.ModeSocket { + return nil + } + + // Follow Symlinks + // Unfortunately, Linux can add bogus symlinks to watch list without + // issue, and Windows can't do symlinks period (AFAIK). To maintain + // consistency, we will act like everything is fine. There will simply + // be no file events for broken symlinks. + // Hence the returns of nil on errors. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + path, err := filepath.EvalSymlinks(path) + if err != nil { + return nil + } + + fi, errstat = os.Lstat(path) + if errstat != nil { + return nil + } + } + + fd, errno := syscall.Open(path, open_FLAGS, 0700) + if fd == -1 { + return errno + } + watchfd = fd + + w.wmut.Lock() + w.watches[path] = watchfd + w.wmut.Unlock() + + w.pmut.Lock() + w.paths[watchfd] = path + w.finfo[watchfd] = fi + w.pmut.Unlock() + } + // Watch the directory if it has not been watched before. + w.pmut.Lock() + w.enmut.Lock() + if w.finfo[watchfd].IsDir() && + (flags&sys_NOTE_WRITE) == sys_NOTE_WRITE && + (!found || (w.enFlags[path]&sys_NOTE_WRITE) != sys_NOTE_WRITE) { + watchDir = true + } + w.enmut.Unlock() + w.pmut.Unlock() + + w.enmut.Lock() + w.enFlags[path] = flags + w.enmut.Unlock() + + var kbuf [1]syscall.Kevent_t + watchEntry := &kbuf[0] + watchEntry.Fflags = flags + syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_ADD|syscall.EV_CLEAR) + entryFlags := watchEntry.Flags + success, errno := syscall.Kevent(w.kq, kbuf[:], nil, nil) + if success == -1 { + return errno + } else if (entryFlags & syscall.EV_ERROR) == syscall.EV_ERROR { + return errors.New("kevent add error") + } + + if watchDir { + errdir := w.watchDirectoryFiles(path) + if errdir != nil { + return errdir + } + } + return nil +} + +// Watch adds path to the watched file set, watching all events. +func (w *Watcher) watch(path string) error { + w.ewmut.Lock() + w.externalWatches[path] = true + w.ewmut.Unlock() + return w.addWatch(path, sys_NOTE_ALLEVENTS) +} + +// RemoveWatch removes path from the watched file set. +func (w *Watcher) removeWatch(path string) error { + w.wmut.Lock() + watchfd, ok := w.watches[path] + w.wmut.Unlock() + if !ok { + return errors.New(fmt.Sprintf("can't remove non-existent kevent watch for: %s", path)) + } + var kbuf [1]syscall.Kevent_t + watchEntry := &kbuf[0] + syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_DELETE) + entryFlags := watchEntry.Flags + success, errno := syscall.Kevent(w.kq, kbuf[:], nil, nil) + if success == -1 { + return os.NewSyscallError("kevent_rm_watch", errno) + } else if (entryFlags & syscall.EV_ERROR) == syscall.EV_ERROR { + return errors.New("kevent rm error") + } + syscall.Close(watchfd) + w.wmut.Lock() + delete(w.watches, path) + w.wmut.Unlock() + w.enmut.Lock() + delete(w.enFlags, path) + w.enmut.Unlock() + w.pmut.Lock() + delete(w.paths, watchfd) + fInfo := w.finfo[watchfd] + delete(w.finfo, watchfd) + w.pmut.Unlock() + + // Find all watched paths that are in this directory that are not external. + if fInfo.IsDir() { + var pathsToRemove []string + w.pmut.Lock() + for _, wpath := range w.paths { + wdir, _ := filepath.Split(wpath) + if filepath.Clean(wdir) == filepath.Clean(path) { + w.ewmut.Lock() + if !w.externalWatches[wpath] { + pathsToRemove = append(pathsToRemove, wpath) + } + w.ewmut.Unlock() + } + } + w.pmut.Unlock() + for _, p := range pathsToRemove { + // Since these are internal, not much sense in propagating error + // to the user, as that will just confuse them with an error about + // a path they did not explicitly watch themselves. + w.removeWatch(p) + } + } + + return nil +} + +// readEvents reads from the kqueue file descriptor, converts the +// received events into Event objects and sends them via the Event channel +func (w *Watcher) readEvents() { + var ( + eventbuf [10]syscall.Kevent_t // Event buffer + events []syscall.Kevent_t // Received events + twait *syscall.Timespec // Time to block waiting for events + n int // Number of events returned from kevent + errno error // Syscall errno + ) + events = eventbuf[0:0] + twait = new(syscall.Timespec) + *twait = syscall.NsecToTimespec(keventWaitTime) + + for { + // See if there is a message on the "done" channel + var done bool + select { + case done = <-w.done: + default: + } + + // If "done" message is received + if done { + errno := syscall.Close(w.kq) + if errno != nil { + w.Error <- os.NewSyscallError("close", errno) + } + close(w.internalEvent) + close(w.Error) + return + } + + // Get new events + if len(events) == 0 { + n, errno = syscall.Kevent(w.kq, nil, eventbuf[:], twait) + + // EINTR is okay, basically the syscall was interrupted before + // timeout expired. + if errno != nil && errno != syscall.EINTR { + w.Error <- os.NewSyscallError("kevent", errno) + continue + } + + // Received some events + if n > 0 { + events = eventbuf[0:n] + } + } + + // Flush the events we received to the events channel + for len(events) > 0 { + fileEvent := new(FileEvent) + watchEvent := &events[0] + fileEvent.mask = uint32(watchEvent.Fflags) + w.pmut.Lock() + fileEvent.Name = w.paths[int(watchEvent.Ident)] + fileInfo := w.finfo[int(watchEvent.Ident)] + w.pmut.Unlock() + if fileInfo != nil && fileInfo.IsDir() && !fileEvent.IsDelete() { + // Double check to make sure the directory exist. This can happen when + // we do a rm -fr on a recursively watched folders and we receive a + // modification event first but the folder has been deleted and later + // receive the delete event + if _, err := os.Lstat(fileEvent.Name); os.IsNotExist(err) { + // mark is as delete event + fileEvent.mask |= sys_NOTE_DELETE + } + } + + if fileInfo != nil && fileInfo.IsDir() && fileEvent.IsModify() && !fileEvent.IsDelete() { + w.sendDirectoryChangeEvents(fileEvent.Name) + } else { + // Send the event on the events channel + w.internalEvent <- fileEvent + } + + // Move to next event + events = events[1:] + + if fileEvent.IsRename() { + w.removeWatch(fileEvent.Name) + w.femut.Lock() + delete(w.fileExists, fileEvent.Name) + w.femut.Unlock() + } + if fileEvent.IsDelete() { + w.removeWatch(fileEvent.Name) + w.femut.Lock() + delete(w.fileExists, fileEvent.Name) + w.femut.Unlock() + + // Look for a file that may have overwritten this + // (ie mv f1 f2 will delete f2 then create f2) + fileDir, _ := filepath.Split(fileEvent.Name) + fileDir = filepath.Clean(fileDir) + w.wmut.Lock() + _, found := w.watches[fileDir] + w.wmut.Unlock() + if found { + // make sure the directory exist before we watch for changes. When we + // do a recursive watch and perform rm -fr, the parent directory might + // have gone missing, ignore the missing directory and let the + // upcoming delete event remove the watch form the parent folder + if _, err := os.Lstat(fileDir); !os.IsNotExist(err) { + w.sendDirectoryChangeEvents(fileDir) + } + } + } + } + } +} + +func (w *Watcher) watchDirectoryFiles(dirPath string) error { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + return err + } + + // Search for new files + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + + // Inherit fsnFlags from parent directory + w.fsnmut.Lock() + if flags, found := w.fsnFlags[dirPath]; found { + w.fsnFlags[filePath] = flags + } else { + w.fsnFlags[filePath] = FSN_ALL + } + w.fsnmut.Unlock() + + if fileInfo.IsDir() == false { + // Watch file to mimic linux fsnotify + e := w.addWatch(filePath, sys_NOTE_ALLEVENTS) + if e != nil { + return e + } + } else { + // If the user is currently watching directory + // we want to preserve the flags used + w.enmut.Lock() + currFlags, found := w.enFlags[filePath] + w.enmut.Unlock() + var newFlags uint32 = sys_NOTE_DELETE + if found { + newFlags |= currFlags + } + + // Linux gives deletes if not explicitly watching + e := w.addWatch(filePath, newFlags) + if e != nil { + return e + } + } + w.femut.Lock() + w.fileExists[filePath] = true + w.femut.Unlock() + } + + return nil +} + +// sendDirectoryEvents searches the directory for newly created files +// and sends them over the event channel. This functionality is to have +// the BSD version of fsnotify match linux fsnotify which provides a +// create event for files created in a watched directory. +func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + w.Error <- err + } + + // Search for new files + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + w.femut.Lock() + _, doesExist := w.fileExists[filePath] + w.femut.Unlock() + if !doesExist { + // Inherit fsnFlags from parent directory + w.fsnmut.Lock() + if flags, found := w.fsnFlags[dirPath]; found { + w.fsnFlags[filePath] = flags + } else { + w.fsnFlags[filePath] = FSN_ALL + } + w.fsnmut.Unlock() + + // Send create event + fileEvent := new(FileEvent) + fileEvent.Name = filePath + fileEvent.create = true + w.internalEvent <- fileEvent + } + w.femut.Lock() + w.fileExists[filePath] = true + w.femut.Unlock() + } + w.watchDirectoryFiles(dirPath) +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_linux.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_linux.go new file mode 100644 index 0000000000000..80ade879fde01 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_linux.go @@ -0,0 +1,304 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "strings" + "sync" + "syscall" + "unsafe" +) + +const ( + // Options for inotify_init() are not exported + // sys_IN_CLOEXEC uint32 = syscall.IN_CLOEXEC + // sys_IN_NONBLOCK uint32 = syscall.IN_NONBLOCK + + // Options for AddWatch + sys_IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW + sys_IN_ONESHOT uint32 = syscall.IN_ONESHOT + sys_IN_ONLYDIR uint32 = syscall.IN_ONLYDIR + + // The "sys_IN_MASK_ADD" option is not exported, as AddWatch + // adds it automatically, if there is already a watch for the given path + // sys_IN_MASK_ADD uint32 = syscall.IN_MASK_ADD + + // Events + sys_IN_ACCESS uint32 = syscall.IN_ACCESS + sys_IN_ALL_EVENTS uint32 = syscall.IN_ALL_EVENTS + sys_IN_ATTRIB uint32 = syscall.IN_ATTRIB + sys_IN_CLOSE uint32 = syscall.IN_CLOSE + sys_IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE + sys_IN_CLOSE_WRITE uint32 = syscall.IN_CLOSE_WRITE + sys_IN_CREATE uint32 = syscall.IN_CREATE + sys_IN_DELETE uint32 = syscall.IN_DELETE + sys_IN_DELETE_SELF uint32 = syscall.IN_DELETE_SELF + sys_IN_MODIFY uint32 = syscall.IN_MODIFY + sys_IN_MOVE uint32 = syscall.IN_MOVE + sys_IN_MOVED_FROM uint32 = syscall.IN_MOVED_FROM + sys_IN_MOVED_TO uint32 = syscall.IN_MOVED_TO + sys_IN_MOVE_SELF uint32 = syscall.IN_MOVE_SELF + sys_IN_OPEN uint32 = syscall.IN_OPEN + + sys_AGNOSTIC_EVENTS = sys_IN_MOVED_TO | sys_IN_MOVED_FROM | sys_IN_CREATE | sys_IN_ATTRIB | sys_IN_MODIFY | sys_IN_MOVE_SELF | sys_IN_DELETE | sys_IN_DELETE_SELF + + // Special events + sys_IN_ISDIR uint32 = syscall.IN_ISDIR + sys_IN_IGNORED uint32 = syscall.IN_IGNORED + sys_IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW + sys_IN_UNMOUNT uint32 = syscall.IN_UNMOUNT +) + +type FileEvent struct { + mask uint32 // Mask of events + cookie uint32 // Unique cookie associating related events (for rename(2)) + Name string // File name (optional) +} + +// IsCreate reports whether the FileEvent was triggered by a creation +func (e *FileEvent) IsCreate() bool { + return (e.mask&sys_IN_CREATE) == sys_IN_CREATE || (e.mask&sys_IN_MOVED_TO) == sys_IN_MOVED_TO +} + +// IsDelete reports whether the FileEvent was triggered by a delete +func (e *FileEvent) IsDelete() bool { + return (e.mask&sys_IN_DELETE_SELF) == sys_IN_DELETE_SELF || (e.mask&sys_IN_DELETE) == sys_IN_DELETE +} + +// IsModify reports whether the FileEvent was triggered by a file modification or attribute change +func (e *FileEvent) IsModify() bool { + return ((e.mask&sys_IN_MODIFY) == sys_IN_MODIFY || (e.mask&sys_IN_ATTRIB) == sys_IN_ATTRIB) +} + +// IsRename reports whether the FileEvent was triggered by a change name +func (e *FileEvent) IsRename() bool { + return ((e.mask&sys_IN_MOVE_SELF) == sys_IN_MOVE_SELF || (e.mask&sys_IN_MOVED_FROM) == sys_IN_MOVED_FROM) +} + +// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata. +func (e *FileEvent) IsAttrib() bool { + return (e.mask & sys_IN_ATTRIB) == sys_IN_ATTRIB +} + +type watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) +} + +type Watcher struct { + mu sync.Mutex // Map access + fd int // File descriptor (as returned by the inotify_init() syscall) + watches map[string]*watch // Map of inotify watches (key: path) + fsnFlags map[string]uint32 // Map of watched files to flags used for filter + fsnmut sync.Mutex // Protects access to fsnFlags. + paths map[int]string // Map of watched paths (key: watch descriptor) + Error chan error // Errors are sent on this channel + internalEvent chan *FileEvent // Events are queued on this channel + Event chan *FileEvent // Events are returned on this channel + done chan bool // Channel for sending a "quit message" to the reader goroutine + isClosed bool // Set to true when Close() is first called +} + +// NewWatcher creates and returns a new inotify instance using inotify_init(2) +func NewWatcher() (*Watcher, error) { + fd, errno := syscall.InotifyInit() + if fd == -1 { + return nil, os.NewSyscallError("inotify_init", errno) + } + w := &Watcher{ + fd: fd, + watches: make(map[string]*watch), + fsnFlags: make(map[string]uint32), + paths: make(map[int]string), + internalEvent: make(chan *FileEvent), + Event: make(chan *FileEvent), + Error: make(chan error), + done: make(chan bool, 1), + } + + go w.readEvents() + go w.purgeEvents() + return w, nil +} + +// Close closes an inotify watcher instance +// It sends a message to the reader goroutine to quit and removes all watches +// associated with the inotify instance +func (w *Watcher) Close() error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Remove all watches + for path := range w.watches { + w.RemoveWatch(path) + } + + // Send "quit" message to the reader goroutine + w.done <- true + + return nil +} + +// AddWatch adds path to the watched file set. +// The flags are interpreted as described in inotify_add_watch(2). +func (w *Watcher) addWatch(path string, flags uint32) error { + if w.isClosed { + return errors.New("inotify instance already closed") + } + + w.mu.Lock() + watchEntry, found := w.watches[path] + w.mu.Unlock() + if found { + watchEntry.flags |= flags + flags |= syscall.IN_MASK_ADD + } + wd, errno := syscall.InotifyAddWatch(w.fd, path, flags) + if wd == -1 { + return errno + } + + w.mu.Lock() + w.watches[path] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = path + w.mu.Unlock() + + return nil +} + +// Watch adds path to the watched file set, watching all events. +func (w *Watcher) watch(path string) error { + return w.addWatch(path, sys_AGNOSTIC_EVENTS) +} + +// RemoveWatch removes path from the watched file set. +func (w *Watcher) removeWatch(path string) error { + w.mu.Lock() + defer w.mu.Unlock() + watch, ok := w.watches[path] + if !ok { + return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path)) + } + success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) + if success == -1 { + return os.NewSyscallError("inotify_rm_watch", errno) + } + delete(w.watches, path) + return nil +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Event channel +func (w *Watcher) readEvents() { + var ( + buf [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + n int // Number of bytes read with read() + errno error // Syscall errno + ) + + for { + // See if there is a message on the "done" channel + select { + case <-w.done: + syscall.Close(w.fd) + close(w.internalEvent) + close(w.Error) + return + default: + } + + n, errno = syscall.Read(w.fd, buf[:]) + + // If EOF is received + if n == 0 { + syscall.Close(w.fd) + close(w.internalEvent) + close(w.Error) + return + } + + if n < 0 { + w.Error <- os.NewSyscallError("read", errno) + continue + } + if n < syscall.SizeofInotifyEvent { + w.Error <- errors.New("inotify: short read in readEvents()") + continue + } + + var offset uint32 = 0 + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-syscall.SizeofInotifyEvent) { + // Point "raw" to the event in the buffer + raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset])) + event := new(FileEvent) + event.mask = uint32(raw.Mask) + event.cookie = uint32(raw.Cookie) + nameLen := uint32(raw.Len) + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + w.mu.Lock() + event.Name = w.paths[int(raw.Wd)] + w.mu.Unlock() + watchedName := event.Name + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent])) + // The filename is padded with NUL bytes. TrimRight() gets rid of those. + event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + // Send the events that are not ignored on the events channel + if !event.ignoreLinux() { + // Setup FSNotify flags (inherit from directory watch) + w.fsnmut.Lock() + if _, fsnFound := w.fsnFlags[event.Name]; !fsnFound { + if fsnFlags, watchFound := w.fsnFlags[watchedName]; watchFound { + w.fsnFlags[event.Name] = fsnFlags + } else { + w.fsnFlags[event.Name] = FSN_ALL + } + } + w.fsnmut.Unlock() + + w.internalEvent <- event + } + + // Move to the next event in the buffer + offset += syscall.SizeofInotifyEvent + nameLen + } + } +} + +// Certain types of events can be "ignored" and not sent over the Event +// channel. Such as events marked ignore by the kernel, or MODIFY events +// against files that do not exist. +func (e *FileEvent) ignoreLinux() bool { + // Ignore anything the inotify API says to ignore + if e.mask&sys_IN_IGNORED == sys_IN_IGNORED { + return true + } + + // If the event is not a DELETE or RENAME, the file must exist. + // Otherwise the event is ignored. + // *Note*: this was put in place because it was seen that a MODIFY + // event was sent after the DELETE. This ignores that MODIFY and + // assumes a DELETE will come or has come if the file doesn't exist. + if !(e.IsDelete() || e.IsRename()) { + _, statErr := os.Lstat(e.Name) + return os.IsNotExist(statErr) + } + return false +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go new file mode 100644 index 0000000000000..f728a6ffd731a --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_bsd.go @@ -0,0 +1,11 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly + +package fsnotify + +import "syscall" + +const open_FLAGS = syscall.O_NONBLOCK | syscall.O_RDONLY diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_darwin.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_darwin.go new file mode 100644 index 0000000000000..d450318e60635 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_open_darwin.go @@ -0,0 +1,11 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package fsnotify + +import "syscall" + +const open_FLAGS = syscall.O_EVTONLY diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_symlink_test.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_symlink_test.go new file mode 100644 index 0000000000000..39061f84426ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_symlink_test.go @@ -0,0 +1,74 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd darwin linux + +package fsnotify + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestFsnotifyFakeSymlink(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + var errorsReceived counter + // Receive errors on the error channel on a separate goroutine + go func() { + for errors := range watcher.Error { + t.Logf("Received error: %s", errors) + errorsReceived.increment() + } + }() + + // Count the CREATE events received + var createEventsReceived, otherEventsReceived counter + go func() { + for ev := range watcher.Event { + t.Logf("event received: %s", ev) + if ev.IsCreate() { + createEventsReceived.increment() + } else { + otherEventsReceived.increment() + } + } + }() + + addWatch(t, watcher, testDir) + + if err := os.Symlink(filepath.Join(testDir, "zzz"), filepath.Join(testDir, "zzznew")); err != nil { + t.Fatalf("Failed to create bogus symlink: %s", err) + } + t.Logf("Created bogus symlink") + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + + // Should not be error, just no events for broken links (watching nothing) + if errorsReceived.value() > 0 { + t.Fatal("fsnotify errors have been received.") + } + if otherEventsReceived.value() > 0 { + t.Fatal("fsnotify other events received on the broken link") + } + + // Except for 1 create event (for the link itself) + if createEventsReceived.value() == 0 { + t.Fatal("fsnotify create events were not received after 500 ms") + } + if createEventsReceived.value() > 1 { + t.Fatal("fsnotify more create events received than expected") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_test.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_test.go new file mode 100644 index 0000000000000..3f5a6487ffe34 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_test.go @@ -0,0 +1,1010 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fsnotify + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" +) + +// An atomic counter +type counter struct { + val int32 +} + +func (c *counter) increment() { + atomic.AddInt32(&c.val, 1) +} + +func (c *counter) value() int32 { + return atomic.LoadInt32(&c.val) +} + +func (c *counter) reset() { + atomic.StoreInt32(&c.val, 0) +} + +// tempMkdir makes a temporary directory +func tempMkdir(t *testing.T) string { + dir, err := ioutil.TempDir("", "fsnotify") + if err != nil { + t.Fatalf("failed to create test directory: %s", err) + } + return dir +} + +// newWatcher initializes an fsnotify Watcher instance. +func newWatcher(t *testing.T) *Watcher { + watcher, err := NewWatcher() + if err != nil { + t.Fatalf("NewWatcher() failed: %s", err) + } + return watcher +} + +// addWatch adds a watch for a directory +func addWatch(t *testing.T, watcher *Watcher, dir string) { + if err := watcher.Watch(dir); err != nil { + t.Fatalf("watcher.Watch(%q) failed: %s", dir, err) + } +} + +func TestFsnotifyMultipleOperations(t *testing.T) { + watcher := newWatcher(t) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory that's not watched + testDirToMoveFiles := tempMkdir(t) + defer os.RemoveAll(testDirToMoveFiles) + + testFile := filepath.Join(testDir, "TestFsnotifySeq.testfile") + testFileRenamed := filepath.Join(testDirToMoveFiles, "TestFsnotifySeqRename.testfile") + + addWatch(t, watcher, testDir) + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var createReceived, modifyReceived, deleteReceived, renameReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + t.Logf("event received: %s", event) + if event.IsDelete() { + deleteReceived.increment() + } + if event.IsModify() { + modifyReceived.increment() + } + if event.IsCreate() { + createReceived.increment() + } + if event.IsRename() { + renameReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // Modify the file outside of the watched dir + f, err = os.Open(testFileRenamed) + if err != nil { + t.Fatalf("open test renamed file failed: %s", err) + } + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Recreate the file that was moved + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Close() + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + mReceived := modifyReceived.value() + if mReceived != 1 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs %d)", mReceived, 1) + } + dReceived := deleteReceived.value() + rReceived := renameReceived.value() + if dReceived+rReceived != 1 { + t.Fatalf("incorrect number of rename+delete events received after 500 ms (%d vs %d)", rReceived+dReceived, 1) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyMultipleCreates(t *testing.T) { + watcher := newWatcher(t) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + testFile := filepath.Join(testDir, "TestFsnotifySeq.testfile") + + addWatch(t, watcher, testDir) + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var createReceived, modifyReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + t.Logf("event received: %s", event) + if event.IsDelete() { + deleteReceived.increment() + } + if event.IsCreate() { + createReceived.increment() + } + if event.IsModify() { + modifyReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + os.Remove(testFile) + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Recreate the file + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Close() + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Modify + f, err = os.OpenFile(testFile, os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Modify + f, err = os.OpenFile(testFile, os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + mReceived := modifyReceived.value() + if mReceived < 3 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs atleast %d)", mReceived, 3) + } + dReceived := deleteReceived.value() + if dReceived != 1 { + t.Fatalf("incorrect number of rename+delete events received after 500 ms (%d vs %d)", dReceived, 1) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyDirOnly(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + // This should NOT add any events to the fsnotify event queue + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyDirOnly.testfile") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var createReceived, modifyReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileAlreadyExists) { + t.Logf("event received: %s", event) + if event.IsDelete() { + deleteReceived.increment() + } + if event.IsModify() { + modifyReceived.increment() + } + if event.IsCreate() { + createReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + os.Remove(testFile) + os.Remove(testFileAlreadyExists) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 1 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 1) + } + mReceived := modifyReceived.value() + if mReceived != 1 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs %d)", mReceived, 1) + } + dReceived := deleteReceived.value() + if dReceived != 2 { + t.Fatalf("incorrect number of delete events received after 500 ms (%d vs %d)", dReceived, 2) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyDeleteWatchedDir(t *testing.T) { + watcher := newWatcher(t) + defer watcher.Close() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + addWatch(t, watcher, testDir) + + // Add a watch for testFile + addWatch(t, watcher, testFileAlreadyExists) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var deleteReceived counter + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFileAlreadyExists) { + t.Logf("event received: %s", event) + if event.IsDelete() { + deleteReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + }() + + os.RemoveAll(testDir) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + dReceived := deleteReceived.value() + if dReceived < 2 { + t.Fatalf("did not receive at least %d delete events, received %d after 500 ms", 2, dReceived) + } +} + +func TestFsnotifySubDir(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + testFile1 := filepath.Join(testDir, "TestFsnotifyFile1.testfile") + testSubDir := filepath.Join(testDir, "sub") + testSubDirFile := filepath.Join(testDir, "sub/TestFsnotifyFile1.testfile") + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var createReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testSubDir) || event.Name == filepath.Clean(testFile1) { + t.Logf("event received: %s", event) + if event.IsCreate() { + createReceived.increment() + } + if event.IsDelete() { + deleteReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + addWatch(t, watcher, testDir) + + // Create sub-directory + if err := os.Mkdir(testSubDir, 0777); err != nil { + t.Fatalf("failed to create test sub-directory: %s", err) + } + + // Create a file + var f *os.File + f, err := os.OpenFile(testFile1, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + // Create a file (Should not see this! we are not watching subdir) + var fs *os.File + fs, err = os.OpenFile(testSubDirFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + fs.Sync() + fs.Close() + + time.Sleep(200 * time.Millisecond) + + // Make sure receive deletes for both file and sub-directory + os.RemoveAll(testSubDir) + os.Remove(testFile1) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + dReceived := deleteReceived.value() + if dReceived != 2 { + t.Fatalf("incorrect number of delete events received after 500 ms (%d vs %d)", dReceived, 2) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyRename(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var renameReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileRenamed) { + if event.IsRename() { + renameReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + f.WriteString("data") + f.Sync() + f.Close() + + // Add a watch for testFile + addWatch(t, watcher, testFile) + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if renameReceived.value() == 0 { + t.Fatal("fsnotify rename events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestFsnotifyRenameToCreate(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory to get file + testDirFrom := tempMkdir(t) + defer os.RemoveAll(testDirFrom) + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDirFrom, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var createReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileRenamed) { + if event.IsCreate() { + createReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if createReceived.value() == 0 { + t.Fatal("fsnotify create events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestFsnotifyRenameToOverwrite(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping test on %q (os.Rename over existing file does not create event).", runtime.GOOS) + } + + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory to get file + testDirFrom := tempMkdir(t) + defer os.RemoveAll(testDirFrom) + + testFile := filepath.Join(testDirFrom, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Create a file + var fr *os.File + fr, err := os.OpenFile(testFileRenamed, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + fr.Sync() + fr.Close() + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + var eventReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testFileRenamed) { + eventReceived.increment() + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if eventReceived.value() == 0 { + t.Fatal("fsnotify events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestRemovalOfWatch(t *testing.T) { + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + watcher := newWatcher(t) + defer watcher.Close() + + addWatch(t, watcher, testDir) + if err := watcher.RemoveWatch(testDir); err != nil { + t.Fatalf("Could not remove the watch: %v\n", err) + } + + go func() { + select { + case ev := <-watcher.Event: + t.Fatalf("We received event: %v\n", ev) + case <-time.After(500 * time.Millisecond): + t.Log("No event received, as expected.") + } + }() + + time.Sleep(200 * time.Millisecond) + // Modify the file outside of the watched dir + f, err := os.Open(testFileAlreadyExists) + if err != nil { + t.Fatalf("Open test file failed: %s", err) + } + f.WriteString("data") + f.Sync() + f.Close() + if err := os.Chmod(testFileAlreadyExists, 0700); err != nil { + t.Fatalf("chmod failed: %s", err) + } + time.Sleep(400 * time.Millisecond) +} + +func TestFsnotifyAttrib(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("attributes don't work on Windows.") + } + + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Error { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyAttrib.testfile") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Event + // The modifyReceived counter counts IsModify events that are not IsAttrib, + // and the attribReceived counts IsAttrib events (which are also IsModify as + // a consequence). + var modifyReceived counter + var attribReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + if event.IsModify() { + modifyReceived.increment() + } + if event.IsAttrib() { + attribReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + f.WriteString("data") + f.Sync() + f.Close() + + // Add a watch for testFile + addWatch(t, watcher, testFile) + + if err := os.Chmod(testFile, 0700); err != nil { + t.Fatalf("chmod failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + // Creating/writing a file changes also the mtime, so IsAttrib should be set to true here + time.Sleep(500 * time.Millisecond) + if modifyReceived.value() == 0 { + t.Fatal("fsnotify modify events have not received after 500 ms") + } + if attribReceived.value() == 0 { + t.Fatal("fsnotify attribute events have not received after 500 ms") + } + + // Modifying the contents of the file does not set the attrib flag (although eg. the mtime + // might have been modified). + modifyReceived.reset() + attribReceived.reset() + + f, err = os.OpenFile(testFile, os.O_WRONLY, 0) + if err != nil { + t.Fatalf("reopening test file failed: %s", err) + } + + f.WriteString("more data") + f.Sync() + f.Close() + + time.Sleep(500 * time.Millisecond) + + if modifyReceived.value() != 1 { + t.Fatal("didn't receive a modify event after changing test file contents") + } + + if attribReceived.value() != 0 { + t.Fatal("did receive an unexpected attrib event after changing test file contents") + } + + modifyReceived.reset() + attribReceived.reset() + + // Doing a chmod on the file should trigger an event with the "attrib" flag set (the contents + // of the file are not changed though) + if err := os.Chmod(testFile, 0600); err != nil { + t.Fatalf("chmod failed: %s", err) + } + + time.Sleep(500 * time.Millisecond) + + if attribReceived.value() != 1 { + t.Fatal("didn't receive an attribute change after 500ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(1e9): + t.Fatal("event stream was not closed after 1 second") + } + + os.Remove(testFile) +} + +func TestFsnotifyClose(t *testing.T) { + watcher := newWatcher(t) + watcher.Close() + + var done int32 + go func() { + watcher.Close() + atomic.StoreInt32(&done, 1) + }() + + time.Sleep(50e6) // 50 ms + if atomic.LoadInt32(&done) == 0 { + t.Fatal("double Close() test failed: second Close() call didn't return") + } + + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + if err := watcher.Watch(testDir); err == nil { + t.Fatal("expected error on Watch() after Close(), got nil") + } +} + +func testRename(file1, file2 string) error { + switch runtime.GOOS { + case "windows", "plan9": + return os.Rename(file1, file2) + default: + cmd := exec.Command("mv", file1, file2) + return cmd.Run() + } +} diff --git a/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_windows.go b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_windows.go new file mode 100644 index 0000000000000..d88ae63407d23 --- /dev/null +++ b/Godeps/_workspace/src/github.com/howeyc/fsnotify/fsnotify_windows.go @@ -0,0 +1,598 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "syscall" + "unsafe" +) + +const ( + // Options for AddWatch + sys_FS_ONESHOT = 0x80000000 + sys_FS_ONLYDIR = 0x1000000 + + // Events + sys_FS_ACCESS = 0x1 + sys_FS_ALL_EVENTS = 0xfff + sys_FS_ATTRIB = 0x4 + sys_FS_CLOSE = 0x18 + sys_FS_CREATE = 0x100 + sys_FS_DELETE = 0x200 + sys_FS_DELETE_SELF = 0x400 + sys_FS_MODIFY = 0x2 + sys_FS_MOVE = 0xc0 + sys_FS_MOVED_FROM = 0x40 + sys_FS_MOVED_TO = 0x80 + sys_FS_MOVE_SELF = 0x800 + + // Special events + sys_FS_IGNORED = 0x8000 + sys_FS_Q_OVERFLOW = 0x4000 +) + +const ( + // TODO(nj): Use syscall.ERROR_MORE_DATA from ztypes_windows in Go 1.3+ + sys_ERROR_MORE_DATA syscall.Errno = 234 +) + +// Event is the type of the notification messages +// received on the watcher's Event channel. +type FileEvent struct { + mask uint32 // Mask of events + cookie uint32 // Unique cookie associating related events (for rename) + Name string // File name (optional) +} + +// IsCreate reports whether the FileEvent was triggered by a creation +func (e *FileEvent) IsCreate() bool { return (e.mask & sys_FS_CREATE) == sys_FS_CREATE } + +// IsDelete reports whether the FileEvent was triggered by a delete +func (e *FileEvent) IsDelete() bool { + return ((e.mask&sys_FS_DELETE) == sys_FS_DELETE || (e.mask&sys_FS_DELETE_SELF) == sys_FS_DELETE_SELF) +} + +// IsModify reports whether the FileEvent was triggered by a file modification or attribute change +func (e *FileEvent) IsModify() bool { + return ((e.mask&sys_FS_MODIFY) == sys_FS_MODIFY || (e.mask&sys_FS_ATTRIB) == sys_FS_ATTRIB) +} + +// IsRename reports whether the FileEvent was triggered by a change name +func (e *FileEvent) IsRename() bool { + return ((e.mask&sys_FS_MOVE) == sys_FS_MOVE || (e.mask&sys_FS_MOVE_SELF) == sys_FS_MOVE_SELF || (e.mask&sys_FS_MOVED_FROM) == sys_FS_MOVED_FROM || (e.mask&sys_FS_MOVED_TO) == sys_FS_MOVED_TO) +} + +// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata. +func (e *FileEvent) IsAttrib() bool { + return (e.mask & sys_FS_ATTRIB) == sys_FS_ATTRIB +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + reply chan error +} + +type inode struct { + handle syscall.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov syscall.Overlapped + ino *inode // i-number + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf [4096]byte +} + +type indexMap map[uint64]*watch +type watchMap map[uint32]indexMap + +// A Watcher waits for and receives event notifications +// for a specific set of files and directories. +type Watcher struct { + mu sync.Mutex // Map access + port syscall.Handle // Handle to completion port + watches watchMap // Map of watches (key: i-number) + fsnFlags map[string]uint32 // Map of watched files to flags used for filter + fsnmut sync.Mutex // Protects access to fsnFlags. + input chan *input // Inputs to the reader are sent on this channel + internalEvent chan *FileEvent // Events are queued on this channel + Event chan *FileEvent // Events are returned on this channel + Error chan error // Errors are sent on this channel + isClosed bool // Set to true when Close() is first called + quit chan chan<- error + cookie uint32 +} + +// NewWatcher creates and returns a Watcher. +func NewWatcher() (*Watcher, error) { + port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if e != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", e) + } + w := &Watcher{ + port: port, + watches: make(watchMap), + fsnFlags: make(map[string]uint32), + input: make(chan *input, 1), + Event: make(chan *FileEvent, 50), + internalEvent: make(chan *FileEvent), + Error: make(chan error), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + go w.purgeEvents() + return w, nil +} + +// Close closes a Watcher. +// It sends a message to the reader goroutine to quit and removes all watches +// associated with the watcher. +func (w *Watcher) Close() error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +// AddWatch adds path to the watched file set. +func (w *Watcher) AddWatch(path string, flags uint32) error { + if w.isClosed { + return errors.New("watcher already closed") + } + in := &input{ + op: opAddWatch, + path: filepath.Clean(path), + flags: flags, + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// Watch adds path to the watched file set, watching all events. +func (w *Watcher) watch(path string) error { + return w.AddWatch(path, sys_FS_ALL_EVENTS) +} + +// RemoveWatch removes path from the watched file set. +func (w *Watcher) removeWatch(path string) error { + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(path), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *Watcher) wakeupReader() error { + e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if e != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", e) + } + return nil +} + +func getDir(pathname string) (dir string, err error) { + attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) + if e != nil { + return "", os.NewSyscallError("GetFileAttributes", e) + } + if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func getIno(path string) (ino *inode, err error) { + h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), + syscall.FILE_LIST_DIRECTORY, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) + if e != nil { + return nil, os.NewSyscallError("CreateFile", e) + } + var fi syscall.ByHandleFileInformation + if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { + syscall.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", e) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *Watcher) addWatch(pathname string, flags uint64) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + if flags&sys_FS_ONLYDIR != 0 && pathname != dir { + return nil + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { + syscall.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", e) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + syscall.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + if err = w.startRead(watchEntry); err != nil { + return err + } + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *Watcher) remWatch(pathname string) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + if watch == nil { + return fmt.Errorf("can't remove non-existent watch for: %s", pathname) + } + if pathname == dir { + w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(watch.path+"\\"+name, watch.names[name]&sys_FS_IGNORED) + delete(watch.names, name) + } + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *Watcher) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(watch.path+"\\"+name, mask&sys_FS_IGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *Watcher) startRead(watch *watch) error { + if e := syscall.CancelIo(watch.ino.handle); e != nil { + w.Error <- os.NewSyscallError("CancelIo", e) + w.deleteWatch(watch) + } + mask := toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= toWindowsFlags(m) + } + if mask == 0 { + if e := syscall.CloseHandle(watch.ino.handle); e != nil { + w.Error <- os.NewSyscallError("CloseHandle", e) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], + uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) + if e != nil { + err := os.NewSyscallError("ReadDirectoryChanges", e) + if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + if w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) { + if watch.mask&sys_FS_ONESHOT != 0 { + watch.mask = 0 + } + } + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Event channel. +// Entry point to the I/O thread. +func (w *Watcher) readEvents() { + var ( + n, key uint32 + ov *syscall.Overlapped + ) + runtime.LockOSThread() + + for { + e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) + watch := (*watch)(unsafe.Pointer(ov)) + + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + var err error + if e := syscall.CloseHandle(w.port); e != nil { + err = os.NewSyscallError("CloseHandle", e) + } + close(w.internalEvent) + close(w.Error) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags)) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch e { + case sys_ERROR_MORE_DATA: + if watch == nil { + w.Error <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case syscall.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case syscall.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.Error <- os.NewSyscallError("GetQueuedCompletionPort", e) + continue + case nil: + } + + var offset uint32 + for { + if n == 0 { + w.internalEvent <- &FileEvent{mask: sys_FS_Q_OVERFLOW} + w.Error <- errors.New("short read in readEvents()") + break + } + + // Point "raw" to the event in the buffer + raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) + name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) + fullname := watch.path + "\\" + name + + var mask uint64 + switch raw.Action { + case syscall.FILE_ACTION_REMOVED: + mask = sys_FS_DELETE_SELF + case syscall.FILE_ACTION_MODIFIED: + mask = sys_FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sys_FS_MOVE_SELF + } + } + + sendNameEvent := func() { + if w.sendEvent(fullname, watch.names[name]&mask) { + if watch.names[name]&sys_FS_ONESHOT != 0 { + delete(watch.names, name) + } + } + } + if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { + sendNameEvent() + } + if raw.Action == syscall.FILE_ACTION_REMOVED { + w.sendEvent(fullname, watch.names[name]&sys_FS_IGNORED) + delete(watch.names, name) + } + if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { + if watch.mask&sys_FS_ONESHOT != 0 { + watch.mask = 0 + } + } + if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { + fullname = watch.path + "\\" + watch.rename + sendNameEvent() + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + w.Error <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") + break + } + } + + if err := w.startRead(watch); err != nil { + w.Error <- err + } + } +} + +func (w *Watcher) sendEvent(name string, mask uint64) bool { + if mask == 0 { + return false + } + event := &FileEvent{mask: uint32(mask), Name: name} + if mask&sys_FS_MOVE != 0 { + if mask&sys_FS_MOVED_FROM != 0 { + w.cookie++ + } + event.cookie = w.cookie + } + select { + case ch := <-w.quit: + w.quit <- ch + case w.Event <- event: + } + return true +} + +func toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sys_FS_ACCESS != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS + } + if mask&sys_FS_MODIFY != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&sys_FS_ATTRIB != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES + } + if mask&(sys_FS_MOVE|sys_FS_CREATE|sys_FS_DELETE) != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func toFSnotifyFlags(action uint32) uint64 { + switch action { + case syscall.FILE_ACTION_ADDED: + return sys_FS_CREATE + case syscall.FILE_ACTION_REMOVED: + return sys_FS_DELETE + case syscall.FILE_ACTION_MODIFIED: + return sys_FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + return sys_FS_MOVED_FROM + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + return sys_FS_MOVED_TO + } + return 0 +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/.gitignore b/Godeps/_workspace/src/github.com/hpcloud/tail/.gitignore new file mode 100644 index 0000000000000..6d9953c3c6ac2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/.gitignore @@ -0,0 +1,3 @@ +.test +.go + diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/.travis.yml b/Godeps/_workspace/src/github.com/hpcloud/tail/.travis.yml new file mode 100644 index 0000000000000..57c00e2096442 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/.travis.yml @@ -0,0 +1,14 @@ +language: go + +# depman needs to be installed with GOPATH pointing to a single directory. +script: + - GOPATH=$HOME/gopath go get github.com/vube/depman + - GOPATH=$HOME/gopath $HOME/gopath/bin/depman + - go test -v ./... + +go: + - 1.3.3 + - 1.4.2 + +install: + - go get gopkg.in/fsnotify.v1 diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/CHANGES.md b/Godeps/_workspace/src/github.com/hpcloud/tail/CHANGES.md new file mode 100644 index 0000000000000..3725d1e24bbb4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/CHANGES.md @@ -0,0 +1,57 @@ +# API current (gopkg.in/ActiveState/tail) + +## July, 2015 + +* Fix inotify watcher leak; remove `Cleanup` (#51) + +# API v0 (gopkg.in/ActiveState/tail.v0) + +## June, 2015 + +* Don't return partial lines (PR #40) +* Use stable version of fsnotify (#46) + +## July, 2014 + +* Fix tail for Windows (PR #36) + +## May, 2014 + +* Improved rate limiting using leaky bucket (PR #29) +* Fix odd line splitting (PR #30) + +## Apr, 2014 + +* LimitRate now discards read buffer (PR #28) +* allow reading of longer lines if MaxLineSize is unset (PR #24) +* updated deps.json to latest fsnotify (441bbc86b1) + +## Feb, 2014 + +* added `Config.Logger` to suppress library logging + +## Nov, 2013 + +* add Cleanup to remove leaky inotify watches (PR #20) + +## Aug, 2013 + +* redesigned Location field (PR #12) +* add tail.Tell (PR #14) + +## July, 2013 + +* Rate limiting (PR #10) + +## May, 2013 + +* Detect file deletions/renames in polling file watcher (PR #1) +* Detect file truncation +* Fix potential race condition when reopening the file (issue 5) +* Fix potential blocking of `tail.Stop` (issue 4) +* Fix uncleaned up ChangeEvents goroutines after calling tail.Stop +* Support Follow=false + +## Feb, 2013 + +* Initial open source release diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/Dockerfile b/Godeps/_workspace/src/github.com/hpcloud/tail/Dockerfile new file mode 100644 index 0000000000000..cd297b940a959 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/Dockerfile @@ -0,0 +1,19 @@ +FROM golang + +RUN mkdir -p $GOPATH/src/github.com/hpcloud/tail/ +ADD . $GOPATH/src/github.com/hpcloud/tail/ + +# expecting to fetch dependencies successfully. +RUN go get -v github.com/hpcloud/tail + +# expecting to run the test successfully. +RUN go test -v github.com/hpcloud/tail + +# expecting to install successfully +RUN go install -v github.com/hpcloud/tail +RUN go install -v github.com/hpcloud/tail/cmd/gotail + +RUN $GOPATH/bin/gotail -h || true + +ENV PATH $GOPATH/bin:$PATH +CMD ["gotail"] diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/LICENSE.txt b/Godeps/_workspace/src/github.com/hpcloud/tail/LICENSE.txt new file mode 100644 index 0000000000000..ea07fde895bfd --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/LICENSE.txt @@ -0,0 +1,23 @@ +# This is the MIT license + +# Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/Makefile b/Godeps/_workspace/src/github.com/hpcloud/tail/Makefile new file mode 100644 index 0000000000000..9ac927db18ea1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/Makefile @@ -0,0 +1,11 @@ +default: test + +test: *.go + go test -v ./... + +fmt: + gofmt -w . + +# Run the test in an isolated environment. +fulltest: + docker build -t hpcloud/tail . diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/README.md b/Godeps/_workspace/src/github.com/hpcloud/tail/README.md new file mode 100644 index 0000000000000..9d93774e84fa4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/README.md @@ -0,0 +1,27 @@ +[![Build Status](https://travis-ci.org/hpcloud/tail.svg)](https://travis-ci.org/ActiveState/tail) + +# Go package for tail-ing files + +A Go package striving to emulate the features of the BSD `tail` program. + +```Go +t, err := tail.TailFile("/var/log/nginx.log", tail.Config{Follow: true}) +for line := range t.Lines { + fmt.Println(line.Text) +} +``` + +See [API documentation](http://godoc.org/github.com/hpcloud/tail). + +## Log rotation + +Tail comes with full support for truncation/move detection as it is +designed to work with log rotation tools. + +## Installing + + go get github.com/hpcloud/tail/... + +## Windows support + +This package [needs assistance](https://github.com/hpcloud/tail/labels/Windows) for full Windows support. diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/.gitignore b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/.gitignore new file mode 100644 index 0000000000000..6249af5fb20ff --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/.gitignore @@ -0,0 +1 @@ +gotail diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/Makefile b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/Makefile new file mode 100644 index 0000000000000..6b5a2d957aa73 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/Makefile @@ -0,0 +1,4 @@ +default: gotail + +gotail: *.go ../../*.go + go build diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/gotail.go b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/gotail.go new file mode 100644 index 0000000000000..c63b582867641 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/cmd/gotail/gotail.go @@ -0,0 +1,64 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package main + +import ( + "flag" + "fmt" + "github.com/hpcloud/tail" + "os" +) + +func args2config() (tail.Config, int64) { + config := tail.Config{Follow: true} + n := int64(0) + maxlinesize := int(0) + flag.Int64Var(&n, "n", 0, "tail from the last Nth location") + flag.IntVar(&maxlinesize, "max", 0, "max line size") + flag.BoolVar(&config.Follow, "f", false, "wait for additional data to be appended to the file") + flag.BoolVar(&config.ReOpen, "F", false, "follow, and track file rename/rotation") + flag.BoolVar(&config.Poll, "p", false, "use polling, instead of inotify") + flag.Parse() + if config.ReOpen { + config.Follow = true + } + config.MaxLineSize = maxlinesize + return config, n +} + +func main() { + config, n := args2config() + if flag.NFlag() < 1 { + fmt.Println("need one or more files as arguments") + os.Exit(1) + } + + if n != 0 { + config.Location = &tail.SeekInfo{-n, os.SEEK_END} + } + + done := make(chan bool) + for _, filename := range flag.Args() { + go tailFile(filename, config, done) + } + + for _, _ = range flag.Args() { + <-done + } +} + +func tailFile(filename string, config tail.Config, done chan bool) { + defer func() { done <- true }() + t, err := tail.TailFile(filename, config) + if err != nil { + fmt.Println(err) + return + } + for line := range t.Lines { + fmt.Println(line.Text) + } + err = t.Wait() + if err != nil { + fmt.Println(err) + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/deps.json b/Godeps/_workspace/src/github.com/hpcloud/tail/deps.json new file mode 100644 index 0000000000000..87e78bdde1ef5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/deps.json @@ -0,0 +1,14 @@ +{ + "github.com/howeyc/fsnotify": { + "repo": "http://github.com/howeyc/fsnotify.git", + "version": "441bbc86b167", + "type": "git-clone", + "alias": "github.com/howeyc/fsnotify" + }, + "gopkg.in/tomb.v1": { + "repo": "https://github.com/go-tomb/tomb.git", + "version": "c131134a1947e9afd9cecfe11f4c6dff0732ae58", + "type": "git-clone", + "alias": "gopkg.in/tomb.v1" + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/Licence b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/Licence new file mode 100644 index 0000000000000..434aab19f1aff --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/Licence @@ -0,0 +1,7 @@ +Copyright (C) 2013 99designs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket.go b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket.go new file mode 100644 index 0000000000000..358b69e7f5c8c --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket.go @@ -0,0 +1,97 @@ +// Package ratelimiter implements the Leaky Bucket ratelimiting algorithm with memcached and in-memory backends. +package ratelimiter + +import ( + "time" +) + +type LeakyBucket struct { + Size uint16 + Fill float64 + LeakInterval time.Duration // time.Duration for 1 unit of size to leak + Lastupdate time.Time + Now func() time.Time +} + +func NewLeakyBucket(size uint16, leakInterval time.Duration) *LeakyBucket { + bucket := LeakyBucket{ + Size: size, + Fill: 0, + LeakInterval: leakInterval, + Now: time.Now, + Lastupdate: time.Now(), + } + + return &bucket +} + +func (b *LeakyBucket) updateFill() { + now := b.Now() + if b.Fill > 0 { + elapsed := now.Sub(b.Lastupdate) + + b.Fill -= float64(elapsed) / float64(b.LeakInterval) + if b.Fill < 0 { + b.Fill = 0 + } + } + b.Lastupdate = now +} + +func (b *LeakyBucket) Pour(amount uint16) bool { + b.updateFill() + + var newfill float64 = b.Fill + float64(amount) + + if newfill > float64(b.Size) { + return false + } + + b.Fill = newfill + + return true +} + +// The time at which this bucket will be completely drained +func (b *LeakyBucket) DrainedAt() time.Time { + return b.Lastupdate.Add(time.Duration(b.Fill * float64(b.LeakInterval))) +} + +// The duration until this bucket is completely drained +func (b *LeakyBucket) TimeToDrain() time.Duration { + return b.DrainedAt().Sub(b.Now()) +} + +func (b *LeakyBucket) TimeSinceLastUpdate() time.Duration { + return b.Now().Sub(b.Lastupdate) +} + +type LeakyBucketSer struct { + Size uint16 + Fill float64 + LeakInterval time.Duration // time.Duration for 1 unit of size to leak + Lastupdate time.Time +} + +func (b *LeakyBucket) Serialise() *LeakyBucketSer { + bucket := LeakyBucketSer{ + Size: b.Size, + Fill: b.Fill, + LeakInterval: b.LeakInterval, + Lastupdate: b.Lastupdate, + } + + return &bucket +} + +func (b *LeakyBucketSer) DeSerialise() *LeakyBucket { + bucket := LeakyBucket{ + Size: b.Size, + Fill: b.Fill, + LeakInterval: b.LeakInterval, + Lastupdate: b.Lastupdate, + Now: time.Now, + } + + return &bucket +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket_test.go b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket_test.go new file mode 100644 index 0000000000000..b43dddb096ae2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/leakybucket_test.go @@ -0,0 +1,73 @@ +package ratelimiter + +import ( + "testing" + "time" +) + +func TestPour(t *testing.T) { + bucket := NewLeakyBucket(60, time.Second) + bucket.Lastupdate = time.Unix(0, 0) + + bucket.Now = func() time.Time { return time.Unix(1, 0) } + + if bucket.Pour(61) { + t.Error("Expected false") + } + + if !bucket.Pour(10) { + t.Error("Expected true") + } + + if !bucket.Pour(49) { + t.Error("Expected true") + } + + if bucket.Pour(2) { + t.Error("Expected false") + } + + bucket.Now = func() time.Time { return time.Unix(61, 0) } + if !bucket.Pour(60) { + t.Error("Expected true") + } + + if bucket.Pour(1) { + t.Error("Expected false") + } + + bucket.Now = func() time.Time { return time.Unix(70, 0) } + + if !bucket.Pour(1) { + t.Error("Expected true") + } + +} + +func TestTimeSinceLastUpdate(t *testing.T) { + bucket := NewLeakyBucket(60, time.Second) + bucket.Now = func() time.Time { return time.Unix(1, 0) } + bucket.Pour(1) + bucket.Now = func() time.Time { return time.Unix(2, 0) } + + sinceLast := bucket.TimeSinceLastUpdate() + if sinceLast != time.Second*1 { + t.Errorf("Expected time since last update to be less than 1 second, got %d", sinceLast) + } +} + +func TestTimeToDrain(t *testing.T) { + bucket := NewLeakyBucket(60, time.Second) + bucket.Now = func() time.Time { return time.Unix(1, 0) } + bucket.Pour(10) + + if bucket.TimeToDrain() != time.Second*10 { + t.Error("Time to drain should be 10 seconds") + } + + bucket.Now = func() time.Time { return time.Unix(2, 0) } + + if bucket.TimeToDrain() != time.Second*9 { + t.Error("Time to drain should be 9 seconds") + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/memory.go b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/memory.go new file mode 100644 index 0000000000000..8f6a5784a9a24 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/memory.go @@ -0,0 +1,58 @@ +package ratelimiter + +import ( + "errors" + "time" +) + +const GC_SIZE int = 100 + +type Memory struct { + store map[string]LeakyBucket + lastGCCollected time.Time +} + +func NewMemory() *Memory { + m := new(Memory) + m.store = make(map[string]LeakyBucket) + m.lastGCCollected = time.Now() + return m +} + +func (m *Memory) GetBucketFor(key string) (*LeakyBucket, error) { + + bucket, ok := m.store[key] + if !ok { + return nil, errors.New("miss") + } + + return &bucket, nil +} + +func (m *Memory) SetBucketFor(key string, bucket LeakyBucket) error { + + if len(m.store) > GC_SIZE { + m.GarbageCollect() + } + + m.store[key] = bucket + + return nil +} + +func (m *Memory) GarbageCollect() { + now := time.Now() + + // rate limit GC to once per minute + if now.Add(60*time.Second).Unix() > m.lastGCCollected.Unix() { + + for key, bucket := range m.store { + // if the bucket is drained, then GC + if bucket.DrainedAt().Unix() > now.Unix() { + delete(m.store, key) + } + } + + m.lastGCCollected = now + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/storage.go b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/storage.go new file mode 100644 index 0000000000000..89b2fe8826e05 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/ratelimiter/storage.go @@ -0,0 +1,6 @@ +package ratelimiter + +type Storage interface { + GetBucketFor(string) (*LeakyBucket, error) + SetBucketFor(string, LeakyBucket) error +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/tail.go b/Godeps/_workspace/src/github.com/hpcloud/tail/tail.go new file mode 100644 index 0000000000000..ae5b95b664ad9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/tail.go @@ -0,0 +1,406 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package tail + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/hpcloud/tail/ratelimiter" + "github.com/hpcloud/tail/util" + "github.com/hpcloud/tail/watch" + "gopkg.in/tomb.v1" +) + +var ( + ErrStop = fmt.Errorf("tail should now stop") +) + +type Line struct { + Text string + Time time.Time + Err error // Error from tail +} + +// NewLine returns a Line with present time. +func NewLine(text string) *Line { + return &Line{text, time.Now(), nil} +} + +// SeekInfo represents arguments to `os.Seek` +type SeekInfo struct { + Offset int64 + Whence int // os.SEEK_* +} + +type logger interface { + Fatal(v ...interface{}) + Fatalf(format string, v ...interface{}) + Fatalln(v ...interface{}) + Panic(v ...interface{}) + Panicf(format string, v ...interface{}) + Panicln(v ...interface{}) + Print(v ...interface{}) + Printf(format string, v ...interface{}) + Println(v ...interface{}) +} + +// Config is used to specify how a file must be tailed. +type Config struct { + // File-specifc + Location *SeekInfo // Seek to this location before tailing + ReOpen bool // Reopen recreated files (tail -F) + MustExist bool // Fail early if the file does not exist + Poll bool // Poll for file changes instead of using inotify + Pipe bool // Is a named pipe (mkfifo) + RateLimiter *ratelimiter.LeakyBucket + + // Generic IO + Follow bool // Continue looking for new lines (tail -f) + MaxLineSize int // If non-zero, split longer lines into multiple lines + + // Logger, when nil, is set to tail.DefaultLogger + // To disable logging: set field to tail.DiscardingLogger + Logger logger +} + +type Tail struct { + Filename string + Lines chan *Line + Config + + file *os.File + reader *bufio.Reader + + watcher watch.FileWatcher + changes *watch.FileChanges + + tomb.Tomb // provides: Done, Kill, Dying +} + +var ( + // DefaultLogger is used when Config.Logger == nil + DefaultLogger = log.New(os.Stderr, "", log.LstdFlags) + // DiscardingLogger can be used to disable logging output + DiscardingLogger = log.New(ioutil.Discard, "", 0) +) + +// TailFile begins tailing the file. Output stream is made available +// via the `Tail.Lines` channel. To handle errors during tailing, +// invoke the `Wait` or `Err` method after finishing reading from the +// `Lines` channel. +func TailFile(filename string, config Config) (*Tail, error) { + if config.ReOpen && !config.Follow { + util.Fatal("cannot set ReOpen without Follow.") + } + + t := &Tail{ + Filename: filename, + Lines: make(chan *Line), + Config: config, + } + + // when Logger was not specified in config, use default logger + if t.Logger == nil { + t.Logger = log.New(os.Stderr, "", log.LstdFlags) + } + + if t.Poll { + t.watcher = watch.NewPollingFileWatcher(filename) + } else { + t.watcher = watch.NewInotifyFileWatcher(filename) + } + + if t.MustExist { + var err error + t.file, err = OpenFile(t.Filename) + if err != nil { + return nil, err + } + } + + go t.tailFileSync() + + return t, nil +} + +// Return the file's current position, like stdio's ftell(). +// But this value is not very accurate. +// it may readed one line in the chan(tail.Lines), +// so it may lost one line. +func (tail *Tail) Tell() (offset int64, err error) { + if tail.file == nil { + return + } + offset, err = tail.file.Seek(0, os.SEEK_CUR) + if err == nil { + offset -= int64(tail.reader.Buffered()) + } + return +} + +// Stop stops the tailing activity. +func (tail *Tail) Stop() error { + tail.Kill(nil) + return tail.Wait() +} + +func (tail *Tail) close() { + close(tail.Lines) + if tail.file != nil { + tail.file.Close() + } +} + +func (tail *Tail) reopen() error { + if tail.file != nil { + tail.file.Close() + } + for { + var err error + tail.file, err = OpenFile(tail.Filename) + if err != nil { + if os.IsNotExist(err) { + tail.Logger.Printf("Waiting for %s to appear...", tail.Filename) + if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil { + if err == tomb.ErrDying { + return err + } + return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err) + } + continue + } + return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err) + } + break + } + return nil +} + +func (tail *Tail) readLine() (string, error) { + line, err := tail.reader.ReadString('\n') + if err != nil { + // Note ReadString "returns the data read before the error" in + // case of an error, including EOF, so we return it as is. The + // caller is expected to process it if err is EOF. + return line, err + } + + line = strings.TrimRight(line, "\n") + + return line, err +} + +func (tail *Tail) tailFileSync() { + defer tail.Done() + defer tail.close() + + if !tail.MustExist { + // deferred first open. + err := tail.reopen() + if err != nil { + if err != tomb.ErrDying { + tail.Kill(err) + } + return + } + } + + // Seek to requested location on first open of the file. + if tail.Location != nil { + _, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence) + tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location) + if err != nil { + tail.Killf("Seek error on %s: %s", tail.Filename, err) + return + } + } + + tail.openReader() + + var offset int64 = 0 + var err error + // Read line by line. + for { + // do not seek in named pipes + if !tail.Pipe { + // grab the position in case we need to back up in the event of a half-line + offset, err = tail.Tell() + if err != nil { + tail.Kill(err) + return + } + } + + line, err := tail.readLine() + + // Process `line` even if err is EOF. + if err == nil { + cooloff := !tail.sendLine(line) + if cooloff { + // Wait a second before seeking till the end of + // file when rate limit is reached. + msg := fmt.Sprintf( + "Too much log activity; waiting a second " + + "before resuming tailing") + tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)} + select { + case <-time.After(time.Second): + case <-tail.Dying(): + return + } + err = tail.seekEnd() + if err != nil { + tail.Kill(err) + return + } + } + } else if err == io.EOF { + if !tail.Follow { + if line != "" { + tail.sendLine(line) + } + return + } + + if tail.Follow && line != "" { + // this has the potential to never return the last line if + // it's not followed by a newline; seems a fair trade here + err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0}) + if err != nil { + tail.Kill(err) + return + } + } + + // When EOF is reached, wait for more data to become + // available. Wait strategy is based on the `tail.watcher` + // implementation (inotify or polling). + err := tail.waitForChanges() + if err != nil { + if err != ErrStop { + tail.Kill(err) + } + return + } + } else { + // non-EOF error + tail.Killf("Error reading %s: %s", tail.Filename, err) + return + } + + select { + case <-tail.Dying(): + return + default: + } + } +} + +// waitForChanges waits until the file has been appended, deleted, +// moved or truncated. When moved or deleted - the file will be +// reopened if ReOpen is true. Truncated files are always reopened. +func (tail *Tail) waitForChanges() error { + if tail.changes == nil { + pos, err := tail.file.Seek(0, os.SEEK_CUR) + if err != nil { + return err + } + tail.changes = tail.watcher.ChangeEvents(&tail.Tomb, pos) + } + + select { + case <-tail.changes.Modified: + return nil + case <-tail.changes.Deleted: + tail.changes = nil + if tail.ReOpen { + // XXX: we must not log from a library. + tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename) + if err := tail.reopen(); err != nil { + return err + } + tail.Logger.Printf("Successfully reopened %s", tail.Filename) + tail.openReader() + return nil + } else { + tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename) + return ErrStop + } + case <-tail.changes.Truncated: + // Always reopen truncated files (Follow is true) + tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename) + if err := tail.reopen(); err != nil { + return err + } + tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename) + tail.openReader() + return nil + case <-tail.Dying(): + return ErrStop + } + panic("unreachable") +} + +func (tail *Tail) openReader() { + if tail.MaxLineSize > 0 { + // add 2 to account for newline characters + tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2) + } else { + tail.reader = bufio.NewReader(tail.file) + } +} + +func (tail *Tail) seekEnd() error { + return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END}) +} + +func (tail *Tail) seekTo(pos SeekInfo) error { + _, err := tail.file.Seek(pos.Offset, pos.Whence) + if err != nil { + return fmt.Errorf("Seek error on %s: %s", tail.Filename, err) + } + // Reset the read buffer whenever the file is re-seek'ed + tail.reader.Reset(tail.file) + return nil +} + +// sendLine sends the line(s) to Lines channel, splitting longer lines +// if necessary. Return false if rate limit is reached. +func (tail *Tail) sendLine(line string) bool { + now := time.Now() + lines := []string{line} + + // Split longer lines + if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize { + lines = util.PartitionString(line, tail.MaxLineSize) + } + + for _, line := range lines { + tail.Lines <- &Line{line, now, nil} + } + + if tail.Config.RateLimiter != nil { + ok := tail.Config.RateLimiter.Pour(uint16(len(lines))) + if !ok { + tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n", + tail.Filename) + return false + } + } + + return true +} + +// Cleanup removes inotify watches added by the tail package. This function is +// meant to be invoked from a process's exit handler. Linux kernel may not +// automatically remove inotify watches after the process exits. +func (tail *Tail) Cleanup() { + watch.Cleanup(tail.Filename) +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/tail_posix.go b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_posix.go new file mode 100644 index 0000000000000..3987b7ef5a82d --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_posix.go @@ -0,0 +1,11 @@ +// +build linux darwin freebsd + +package tail + +import ( + "os" +) + +func OpenFile(name string) (file *os.File, err error) { + return os.Open(name) +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/tail_test.go b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_test.go new file mode 100644 index 0000000000000..b2f22a5978d86 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_test.go @@ -0,0 +1,438 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +// TODO: +// * repeat all the tests with Poll:true + +package tail + +import ( + _ "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "./watch" + "github.com/hpcloud/tail/ratelimiter" +) + +func init() { + // Clear the temporary test directory + err := os.RemoveAll(".test") + if err != nil { + panic(err) + } +} + +func TestMustExist(t *testing.T) { + tail, err := TailFile("/no/such/file", Config{Follow: true, MustExist: true}) + if err == nil { + t.Error("MustExist:true is violated") + tail.Stop() + } + tail, err = TailFile("/no/such/file", Config{Follow: true, MustExist: false}) + if err != nil { + t.Error("MustExist:false is violated") + } + tail.Stop() + _, err = TailFile("README.md", Config{Follow: true, MustExist: true}) + if err != nil { + t.Error("MustExist:true on an existing file is violated") + } + tail.Stop() + tail.Cleanup() +} + +func TestStop(t *testing.T) { + tail, err := TailFile("_no_such_file", Config{Follow: true, MustExist: false}) + if err != nil { + t.Error("MustExist:false is violated") + } + if tail.Stop() != nil { + t.Error("Should be stoped successfully") + } + tail.Cleanup() +} + +func MaxLineSizeT(_t *testing.T, follow bool, fileContent string, expected []string) { + t := NewTailTest("maxlinesize", _t) + t.CreateFile("test.txt", fileContent) + tail := t.StartTail("test.txt", Config{Follow: follow, Location: nil, MaxLineSize: 3}) + go t.VerifyTailOutput(tail, expected) + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + tail.Stop() + tail.Cleanup() +} + +func TestMaxLineSizeFollow(_t *testing.T) { + // As last file line does not end with newline, it will not be present in tail's output + MaxLineSizeT(_t, true, "hello\nworld\nfin\nhe", []string{"hel", "lo", "wor", "ld", "fin"}) +} + +func TestMaxLineSizeNoFollow(_t *testing.T) { + MaxLineSizeT(_t, false, "hello\nworld\nfin\nhe", []string{"hel", "lo", "wor", "ld", "fin", "he"}) +} + +func TestOver4096ByteLine(_t *testing.T) { + t := NewTailTest("Over4096ByteLine", _t) + testString := strings.Repeat("a", 4097) + t.CreateFile("test.txt", "test\n"+testString+"\nhello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: true, Location: nil}) + go t.VerifyTailOutput(tail, []string{"test", testString, "hello", "world"}) + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + tail.Stop() + tail.Cleanup() +} +func TestOver4096ByteLineWithSetMaxLineSize(_t *testing.T) { + t := NewTailTest("Over4096ByteLineMaxLineSize", _t) + testString := strings.Repeat("a", 4097) + t.CreateFile("test.txt", "test\n"+testString+"\nhello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: true, Location: nil, MaxLineSize: 4097}) + go t.VerifyTailOutput(tail, []string{"test", testString, "hello", "world"}) + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + // tail.Stop() + tail.Cleanup() +} + +func TestLocationFull(_t *testing.T) { + t := NewTailTest("location-full", _t) + t.CreateFile("test.txt", "hello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: true, Location: nil}) + go t.VerifyTailOutput(tail, []string{"hello", "world"}) + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + tail.Stop() + tail.Cleanup() +} + +func TestLocationFullDontFollow(_t *testing.T) { + t := NewTailTest("location-full-dontfollow", _t) + t.CreateFile("test.txt", "hello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: false, Location: nil}) + go t.VerifyTailOutput(tail, []string{"hello", "world"}) + + // Add more data only after reasonable delay. + <-time.After(100 * time.Millisecond) + t.AppendFile("test.txt", "more\ndata\n") + <-time.After(100 * time.Millisecond) + + tail.Stop() + tail.Cleanup() +} + +func TestLocationEnd(_t *testing.T) { + t := NewTailTest("location-end", _t) + t.CreateFile("test.txt", "hello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: true, Location: &SeekInfo{0, os.SEEK_END}}) + go t.VerifyTailOutput(tail, []string{"more", "data"}) + + <-time.After(100 * time.Millisecond) + t.AppendFile("test.txt", "more\ndata\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + tail.Stop() + tail.Cleanup() +} + +func TestLocationMiddle(_t *testing.T) { + // Test reading from middle. + t := NewTailTest("location-end", _t) + t.CreateFile("test.txt", "hello\nworld\n") + tail := t.StartTail("test.txt", Config{Follow: true, Location: &SeekInfo{-6, os.SEEK_END}}) + go t.VerifyTailOutput(tail, []string{"world", "more", "data"}) + + <-time.After(100 * time.Millisecond) + t.AppendFile("test.txt", "more\ndata\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + tail.Stop() + tail.Cleanup() +} + +func _TestReOpen(_t *testing.T, poll bool) { + var name string + var delay time.Duration + if poll { + name = "reopen-polling" + delay = 300 * time.Millisecond // account for POLL_DURATION + } else { + name = "reopen-inotify" + delay = 100 * time.Millisecond + } + t := NewTailTest(name, _t) + t.CreateFile("test.txt", "hello\nworld\n") + tail := t.StartTail( + "test.txt", + Config{Follow: true, ReOpen: true, Poll: poll}) + + go t.VerifyTailOutput(tail, []string{"hello", "world", "more", "data", "endofworld"}) + + // deletion must trigger reopen + <-time.After(delay) + t.RemoveFile("test.txt") + <-time.After(delay) + t.CreateFile("test.txt", "more\ndata\n") + + // rename must trigger reopen + <-time.After(delay) + t.RenameFile("test.txt", "test.txt.rotated") + <-time.After(delay) + t.CreateFile("test.txt", "endofworld") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(delay) + t.RemoveFile("test.txt") + <-time.After(delay) + + // Do not bother with stopping as it could kill the tomb during + // the reading of data written above. Timings can vary based on + // test environment. + // tail.Stop() + tail.Cleanup() +} + +// The use of polling file watcher could affect file rotation +// (detected via renames), so test these explicitly. + +func TestReOpenInotify(_t *testing.T) { + _TestReOpen(_t, false) +} + +func TestReOpenPolling(_t *testing.T) { + _TestReOpen(_t, true) +} + +func _TestReSeek(_t *testing.T, poll bool) { + var name string + if poll { + name = "reseek-polling" + } else { + name = "reseek-inotify" + } + t := NewTailTest(name, _t) + t.CreateFile("test.txt", "a really long string goes here\nhello\nworld\n") + tail := t.StartTail( + "test.txt", + Config{Follow: true, ReOpen: false, Poll: poll}) + + go t.VerifyTailOutput(tail, []string{ + "a really long string goes here", "hello", "world", "h311o", "w0r1d", "endofworld"}) + + // truncate now + <-time.After(100 * time.Millisecond) + t.TruncateFile("test.txt", "h311o\nw0r1d\nendofworld\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + + // Do not bother with stopping as it could kill the tomb during + // the reading of data written above. Timings can vary based on + // test environment. + // tail.Stop() + tail.Cleanup() +} + +// The use of polling file watcher could affect file rotation +// (detected via renames), so test these explicitly. + +func TestReSeekInotify(_t *testing.T) { + _TestReSeek(_t, false) +} + +func TestReSeekPolling(_t *testing.T) { + _TestReSeek(_t, true) +} + +func TestRateLimiting(_t *testing.T) { + t := NewTailTest("rate-limiting", _t) + t.CreateFile("test.txt", "hello\nworld\nagain\nextra\n") + config := Config{ + Follow: true, + RateLimiter: ratelimiter.NewLeakyBucket(2, time.Second)} + leakybucketFull := "Too much log activity; waiting a second before resuming tailing" + tail := t.StartTail("test.txt", config) + + // TODO: also verify that tail resumes after the cooloff period. + go t.VerifyTailOutput(tail, []string{ + "hello", "world", "again", + leakybucketFull, + "more", "data", + leakybucketFull}) + + // Add more data only after reasonable delay. + <-time.After(1200 * time.Millisecond) + t.AppendFile("test.txt", "more\ndata\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + t.RemoveFile("test.txt") + + // tail.Stop() + tail.Cleanup() +} + +func TestTell(_t *testing.T) { + t := NewTailTest("tell-position", _t) + t.CreateFile("test.txt", "hello\nworld\nagain\nmore\n") + config := Config{ + Follow: false, + Location: &SeekInfo{0, os.SEEK_SET}} + tail := t.StartTail("test.txt", config) + // read noe line + <-tail.Lines + offset, err := tail.Tell() + if err != nil { + t.Errorf("Tell return error: %s", err.Error()) + } + tail.Done() + // tail.close() + + config = Config{ + Follow: false, + Location: &SeekInfo{offset, os.SEEK_SET}} + tail = t.StartTail("test.txt", config) + for l := range tail.Lines { + // it may readed one line in the chan(tail.Lines), + // so it may lost one line. + if l.Text != "world" && l.Text != "again" { + t.Fatalf("mismatch; expected world or again, but got %s", + l.Text) + } + break + } + t.RemoveFile("test.txt") + tail.Done() + tail.Cleanup() +} + +// Test library + +type TailTest struct { + Name string + path string + *testing.T +} + +func NewTailTest(name string, t *testing.T) TailTest { + tt := TailTest{name, ".test/" + name, t} + err := os.MkdirAll(tt.path, os.ModeTemporary|0700) + if err != nil { + tt.Fatal(err) + } + + // Use a smaller poll duration for faster test runs. Keep it below + // 100ms (which value is used as common delays for tests) + watch.POLL_DURATION = 5 * time.Millisecond + + return tt +} + +func (t TailTest) CreateFile(name string, contents string) { + err := ioutil.WriteFile(t.path+"/"+name, []byte(contents), 0600) + if err != nil { + t.Fatal(err) + } +} + +func (t TailTest) RemoveFile(name string) { + err := os.Remove(t.path + "/" + name) + if err != nil { + t.Fatal(err) + } +} + +func (t TailTest) RenameFile(oldname string, newname string) { + oldname = t.path + "/" + oldname + newname = t.path + "/" + newname + err := os.Rename(oldname, newname) + if err != nil { + t.Fatal(err) + } +} + +func (t TailTest) AppendFile(name string, contents string) { + f, err := os.OpenFile(t.path+"/"+name, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer f.Close() + _, err = f.WriteString(contents) + if err != nil { + t.Fatal(err) + } +} + +func (t TailTest) TruncateFile(name string, contents string) { + f, err := os.OpenFile(t.path+"/"+name, os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer f.Close() + _, err = f.WriteString(contents) + if err != nil { + t.Fatal(err) + } +} + +func (t TailTest) StartTail(name string, config Config) *Tail { + tail, err := TailFile(t.path+"/"+name, config) + if err != nil { + t.Fatal(err) + } + return tail +} + +func (t TailTest) VerifyTailOutput(tail *Tail, lines []string) { + for idx, line := range lines { + tailedLine, ok := <-tail.Lines + if !ok { + // tail.Lines is closed and empty. + err := tail.Err() + if err != nil { + t.Fatalf("tail ended with error: %v", err) + } + t.Fatalf("tail ended early; expecting more: %v", lines[idx:]) + } + if tailedLine == nil { + t.Fatalf("tail.Lines returned nil; not possible") + } + // Note: not checking .Err as the `lines` argument is designed + // to match error strings as well. + if tailedLine.Text != line { + t.Fatalf( + "unexpected line/err from tail: "+ + "expecting <<%s>>>, but got <<<%s>>>", + line, tailedLine.Text) + } + } + line, ok := <-tail.Lines + if ok { + t.Fatalf("more content from tail: %+v", line) + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/tail_windows.go b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_windows.go new file mode 100644 index 0000000000000..ef2cfca1b74b4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/tail_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package tail + +import ( + "github.com/hpcloud/tail/winfile" + "os" +) + +func OpenFile(name string) (file *os.File, err error) { + return winfile.OpenFile(name, os.O_RDONLY, 0) +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/util/util.go b/Godeps/_workspace/src/github.com/hpcloud/tail/util/util.go new file mode 100644 index 0000000000000..f58b3ee9d7029 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/util/util.go @@ -0,0 +1,47 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package util + +import ( + "fmt" + "log" + "os" + "runtime/debug" +) + +type Logger struct { + *log.Logger +} + +var LOGGER = &Logger{log.New(os.Stderr, "", log.LstdFlags)} + +// fatal is like panic except it displays only the current goroutine's stack. +func Fatal(format string, v ...interface{}) { + // https://github.com/ActiveState/log/blob/master/log.go#L45 + LOGGER.Output(2, fmt.Sprintf("FATAL -- "+format, v...)+"\n"+string(debug.Stack())) + os.Exit(1) +} + +// partitionString partitions the string into chunks of given size, +// with the last chunk of variable size. +func PartitionString(s string, chunkSize int) []string { + if chunkSize <= 0 { + panic("invalid chunkSize") + } + length := len(s) + chunks := 1 + length/chunkSize + start := 0 + end := chunkSize + parts := make([]string, 0, chunks) + for { + if end > length { + end = length + } + parts = append(parts, s[start:end]) + if end == length { + break + } + start, end = end, end+chunkSize + } + return parts +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/watch/filechanges.go b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/filechanges.go new file mode 100644 index 0000000000000..fb0f9ef5361b7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/filechanges.go @@ -0,0 +1,42 @@ +package watch + +type FileChanges struct { + Modified chan bool // Channel to get notified of modifications + Truncated chan bool // Channel to get notified of truncations + Deleted chan bool // Channel to get notified of deletions/renames +} + +func NewFileChanges() *FileChanges { + return &FileChanges{ + make(chan bool), make(chan bool), make(chan bool)} +} + +func (fc *FileChanges) NotifyModified() { + sendOnlyIfEmpty(fc.Modified) +} + +func (fc *FileChanges) NotifyTruncated() { + sendOnlyIfEmpty(fc.Truncated) +} + +func (fc *FileChanges) NotifyDeleted() { + sendOnlyIfEmpty(fc.Deleted) +} + +func (fc *FileChanges) Close() { + close(fc.Modified) + close(fc.Truncated) + close(fc.Deleted) +} + +// sendOnlyIfEmpty sends on a bool channel only if the channel has no +// backlog to be read by other goroutines. This concurrency pattern +// can be used to notify other goroutines if and only if they are +// looking for it (i.e., subsequent notifications can be compressed +// into one). +func sendOnlyIfEmpty(ch chan bool) { + select { + case ch <- true: + default: + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify.go b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify.go new file mode 100644 index 0000000000000..d6635f4b3f0c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify.go @@ -0,0 +1,123 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package watch + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hpcloud/tail/util" + + "gopkg.in/fsnotify.v1" + "gopkg.in/tomb.v1" +) + +// InotifyFileWatcher uses inotify to monitor file changes. +type InotifyFileWatcher struct { + Filename string + Size int64 +} + +func NewInotifyFileWatcher(filename string) *InotifyFileWatcher { + fw := &InotifyFileWatcher{filename, 0} + return fw +} + +func (fw *InotifyFileWatcher) BlockUntilExists(t *tomb.Tomb) error { + dirname := filepath.Dir(fw.Filename) + + // Watch for new files to be created in the parent directory. + err := Watch(dirname) + if err != nil { + return err + } + defer RemoveWatch(dirname) + + // Do a real check now as the file might have been created before + // calling `WatchFlags` above. + if _, err = os.Stat(fw.Filename); !os.IsNotExist(err) { + // file exists, or stat returned an error. + return err + } + + events := Events(fw.Filename) + + for { + select { + case evt, ok := <-events: + if !ok { + return fmt.Errorf("inotify watcher has been closed") + } else if evt.Name == fw.Filename { + return nil + } + case <-t.Dying(): + return tomb.ErrDying + } + } + panic("unreachable") +} + +func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) *FileChanges { + changes := NewFileChanges() + + err := Watch(fw.Filename) + if err != nil { + go changes.NotifyDeleted() + } + + fw.Size = pos + + go func() { + defer RemoveWatch(fw.Filename) + defer changes.Close() + + events := Events(fw.Filename) + + for { + prevSize := fw.Size + + var evt fsnotify.Event + var ok bool + + select { + case evt, ok = <-events: + if !ok { + return + } + case <-t.Dying(): + return + } + + switch { + case evt.Op&fsnotify.Remove == fsnotify.Remove: + fallthrough + + case evt.Op&fsnotify.Rename == fsnotify.Rename: + changes.NotifyDeleted() + return + + case evt.Op&fsnotify.Write == fsnotify.Write: + fi, err := os.Stat(fw.Filename) + if err != nil { + if os.IsNotExist(err) { + changes.NotifyDeleted() + return + } + // XXX: report this error back to the user + util.Fatal("Failed to stat file %v: %v", fw.Filename, err) + } + fw.Size = fi.Size() + + if prevSize > 0 && prevSize > fw.Size { + changes.NotifyTruncated() + } else { + changes.NotifyModified() + } + prevSize = fw.Size + } + } + }() + + return changes +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify_tracker.go b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify_tracker.go new file mode 100644 index 0000000000000..07e7d7a103fd9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/inotify_tracker.go @@ -0,0 +1,169 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package watch + +import ( + "log" + "os" + "sync" + "syscall" + + "github.com/hpcloud/tail/util" + + "gopkg.in/fsnotify.v1" +) + +type InotifyTracker struct { + mux sync.Mutex + watcher *fsnotify.Watcher + chans map[string]chan fsnotify.Event + done map[string]chan bool + watch chan *watchInfo + remove chan string + error chan error +} + +type watchInfo struct { + fname string +} + +var ( + // globally shared InotifyTracker; ensures only one fsnotify.Watcher is used + shared *InotifyTracker + + // these are used to ensure the shared InotifyTracker is run exactly once + once = sync.Once{} + goRun = func() { + shared = &InotifyTracker{ + mux: sync.Mutex{}, + chans: make(map[string]chan fsnotify.Event), + done: make(map[string]chan bool), + watch: make(chan *watchInfo), + remove: make(chan string), + error: make(chan error), + } + go shared.run() + } + + logger = log.New(os.Stderr, "", log.LstdFlags) +) + +// Watch signals the run goroutine to begin watching the input filename +func Watch(fname string) error { + // start running the shared InotifyTracker if not already running + once.Do(goRun) + + shared.watch <- &watchInfo{ + fname: fname, + } + return <-shared.error +} + +// RemoveWatch signals the run goroutine to remove the watch for the input filename +func RemoveWatch(fname string) { + // start running the shared InotifyTracker if not already running + once.Do(goRun) + + shared.mux.Lock() + done := shared.done[fname] + if done != nil { + delete(shared.done, fname) + close(done) + } + shared.mux.Unlock() + + shared.remove <- fname +} + +// Events returns a channel to which FileEvents corresponding to the input filename +// will be sent. This channel will be closed when removeWatch is called on this +// filename. +func Events(fname string) chan fsnotify.Event { + shared.mux.Lock() + defer shared.mux.Unlock() + + return shared.chans[fname] +} + +// Cleanup removes the watch for the input filename if necessary. +func Cleanup(fname string) { + RemoveWatch(fname) +} + +// watchFlags calls fsnotify.WatchFlags for the input filename and flags, creating +// a new Watcher if the previous Watcher was closed. +func (shared *InotifyTracker) addWatch(fname string) error { + shared.mux.Lock() + defer shared.mux.Unlock() + + if shared.chans[fname] == nil { + shared.chans[fname] = make(chan fsnotify.Event) + shared.done[fname] = make(chan bool) + } + return shared.watcher.Add(fname) +} + +// removeWatch calls fsnotify.RemoveWatch for the input filename and closes the +// corresponding events channel. +func (shared *InotifyTracker) removeWatch(fname string) { + shared.mux.Lock() + defer shared.mux.Unlock() + + if ch := shared.chans[fname]; ch != nil { + shared.watcher.Remove(fname) + + delete(shared.chans, fname) + close(ch) + } +} + +// sendEvent sends the input event to the appropriate Tail. +func (shared *InotifyTracker) sendEvent(event fsnotify.Event) { + shared.mux.Lock() + ch := shared.chans[event.Name] + done := shared.done[event.Name] + shared.mux.Unlock() + + if ch != nil && done != nil { + select { + case ch <- event: + case <-done: + } + } +} + +// run starts the goroutine in which the shared struct reads events from its +// Watcher's Event channel and sends the events to the appropriate Tail. +func (shared *InotifyTracker) run() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + util.Fatal("failed to create Watcher") + } + shared.watcher = watcher + + for { + select { + case winfo := <-shared.watch: + shared.error <- shared.addWatch(winfo.fname) + + case fname := <-shared.remove: + shared.removeWatch(fname) + + case event, open := <-shared.watcher.Events: + if !open { + return + } + shared.sendEvent(event) + + case err, open := <-shared.watcher.Errors: + if !open { + return + } else if err != nil { + sysErr, ok := err.(*os.SyscallError) + if !ok || sysErr.Err != syscall.EINTR { + logger.Printf("Error in Watcher Error channel: %s", err) + } + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/watch/polling.go b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/polling.go new file mode 100644 index 0000000000000..5479272a69ffc --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/polling.go @@ -0,0 +1,118 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package watch + +import ( + "github.com/hpcloud/tail/util" + "gopkg.in/tomb.v1" + "os" + "runtime" + "time" +) + +// PollingFileWatcher polls the file for changes. +type PollingFileWatcher struct { + Filename string + Size int64 +} + +func NewPollingFileWatcher(filename string) *PollingFileWatcher { + fw := &PollingFileWatcher{filename, 0} + return fw +} + +var POLL_DURATION time.Duration + +func (fw *PollingFileWatcher) BlockUntilExists(t *tomb.Tomb) error { + for { + if _, err := os.Stat(fw.Filename); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + select { + case <-time.After(POLL_DURATION): + continue + case <-t.Dying(): + return tomb.ErrDying + } + } + panic("unreachable") +} + +func (fw *PollingFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) *FileChanges { + changes := NewFileChanges() + var prevModTime time.Time + + // XXX: use tomb.Tomb to cleanly manage these goroutines. replace + // the fatal (below) with tomb's Kill. + + fw.Size = pos + origFi, err := os.Stat(fw.Filename) + if err != nil { + changes.NotifyDeleted() + return changes + } + + go func() { + defer changes.Close() + + prevSize := fw.Size + for { + select { + case <-t.Dying(): + return + default: + } + + time.Sleep(POLL_DURATION) + fi, err := os.Stat(fw.Filename) + if err != nil { + // Windows cannot delete a file if a handle is still open (tail keeps one open) + // so it gives access denied to anything trying to read it until all handles are released. + if os.IsNotExist(err) || (runtime.GOOS == "windows" && os.IsPermission(err)) { + // File does not exist (has been deleted). + changes.NotifyDeleted() + return + } + + // XXX: report this error back to the user + util.Fatal("Failed to stat file %v: %v", fw.Filename, err) + } + + // File got moved/renamed? + if !os.SameFile(origFi, fi) { + changes.NotifyDeleted() + return + } + + // File got truncated? + fw.Size = fi.Size() + if prevSize > 0 && prevSize > fw.Size { + changes.NotifyTruncated() + prevSize = fw.Size + continue + } + // File got bigger? + if prevSize > 0 && prevSize < fw.Size { + changes.NotifyModified() + prevSize = fw.Size + continue + } + prevSize = fw.Size + + // File was appended to (changed)? + modTime := fi.ModTime() + if modTime != prevModTime { + prevModTime = modTime + changes.NotifyModified() + } + } + }() + + return changes +} + +func init() { + POLL_DURATION = 250 * time.Millisecond +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/watch/watch.go b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/watch.go new file mode 100644 index 0000000000000..fbc08a5c5e349 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/watch/watch.go @@ -0,0 +1,19 @@ +// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. + +package watch + +import "gopkg.in/tomb.v1" + +// FileWatcher monitors file-level events. +type FileWatcher interface { + // BlockUntilExists blocks until the file comes into existence. + BlockUntilExists(*tomb.Tomb) error + + // ChangeEvents reports on changes to a file, be it modification, + // deletion, renames or truncations. Returned FileChanges group of + // channels will be closed, thus become unusable, after a deletion + // or truncation event. + // In order to properly report truncations, ChangeEvents requires + // the caller to pass their current offset in the file. + ChangeEvents(*tomb.Tomb, int64) *FileChanges +} diff --git a/Godeps/_workspace/src/github.com/hpcloud/tail/winfile/winfile.go b/Godeps/_workspace/src/github.com/hpcloud/tail/winfile/winfile.go new file mode 100644 index 0000000000000..aa7e7bc5df597 --- /dev/null +++ b/Godeps/_workspace/src/github.com/hpcloud/tail/winfile/winfile.go @@ -0,0 +1,92 @@ +// +build windows + +package winfile + +import ( + "os" + "syscall" + "unsafe" +) + +// issue also described here +//https://codereview.appspot.com/8203043/ + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L218 +func Open(path string, mode int, perm uint32) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_CREAT != 0 { + access |= syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) + var sa *syscall.SecurityAttributes + if mode&syscall.O_CLOEXEC == 0 { + sa = makeInheritSa() + } + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) + return h, e +} + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L211 +func makeInheritSa() *syscall.SecurityAttributes { + var sa syscall.SecurityAttributes + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + return &sa +} + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/os/file_windows.go#L133 +func OpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) { + r, e := Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm)) + if e != nil { + return nil, e + } + return os.NewFile(uintptr(r), name), nil +} + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/os/file_posix.go#L61 +func syscallMode(i os.FileMode) (o uint32) { + o |= uint32(i.Perm()) + if i&os.ModeSetuid != 0 { + o |= syscall.S_ISUID + } + if i&os.ModeSetgid != 0 { + o |= syscall.S_ISGID + } + if i&os.ModeSticky != 0 { + o |= syscall.S_ISVTX + } + // No mapping for Go's ModeTemporary (plan9 only). + return +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.gitignore b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.gitignore new file mode 100644 index 0000000000000..4cd0cbaf432cc --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.gitignore @@ -0,0 +1,6 @@ +# Setup a Global .gitignore for OS and editor generated files: +# https://help.github.com/articles/ignoring-files +# git config --global core.excludesfile ~/.gitignore_global + +.vagrant +*.sublime-project diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml new file mode 100644 index 0000000000000..1b5151f123900 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/.travis.yml @@ -0,0 +1,20 @@ +sudo: false +language: go + +go: + - 1.5.1 + +before_script: + - go get -u github.com/golang/lint/golint + +after_script: + - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" + - test -z "$(golint ./... | tee /dev/stderr)" + - go vet ./... + +os: + - linux + - osx + +notifications: + email: false diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS new file mode 100644 index 0000000000000..4e0e8284e95dc --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/AUTHORS @@ -0,0 +1,34 @@ +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# You can update this list using the following command: +# +# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' + +# Please keep the list sorted. + +Adrien Bustany +Caleb Spare +Case Nelson +Chris Howey +Christoffer Buchholz +Dave Cheney +Francisco Souza +Hari haran +John C Barstow +Kelvin Fo +Matt Layher +Nathan Youngman +Paul Hammond +Pieter Droogendijk +Pursuit92 +Rob Figueiredo +Soge Zhang +Tilak Sharma +Travis Cline +Tudor Golubenco +Yukang +bronze1man +debrando +henrikedwards diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md new file mode 100644 index 0000000000000..b0bfea07c3e8a --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CHANGELOG.md @@ -0,0 +1,267 @@ +# Changelog + +## v1.2.1 / 2015-10-14 + +* kqueue: don't watch named pipes [#98](https://github.com/go-fsnotify/fsnotify/pull/98) (thanks @evanphx) + +## v1.2.0 / 2015-02-08 + +* inotify: use epoll to wake up readEvents [#66](https://github.com/go-fsnotify/fsnotify/pull/66) (thanks @PieterD) +* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/go-fsnotify/fsnotify/pull/63) (thanks @PieterD) +* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/go-fsnotify/fsnotify/issues/59) + +## v1.1.1 / 2015-02-05 + +* inotify: Retry read on EINTR [#61](https://github.com/go-fsnotify/fsnotify/issues/61) (thanks @PieterD) + +## v1.1.0 / 2014-12-12 + +* kqueue: rework internals [#43](https://github.com/go-fsnotify/fsnotify/pull/43) + * add low-level functions + * only need to store flags on directories + * less mutexes [#13](https://github.com/go-fsnotify/fsnotify/issues/13) + * done can be an unbuffered channel + * remove calls to os.NewSyscallError +* More efficient string concatenation for Event.String() [#52](https://github.com/go-fsnotify/fsnotify/pull/52) (thanks @mdlayher) +* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/go-fsnotify/fsnotify/issues/48) +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/go-fsnotify/fsnotify/issues/51) + +## v1.0.4 / 2014-09-07 + +* kqueue: add dragonfly to the build tags. +* Rename source code files, rearrange code so exported APIs are at the top. +* Add done channel to example code. [#37](https://github.com/go-fsnotify/fsnotify/pull/37) (thanks @chenyukang) + +## v1.0.3 / 2014-08-19 + +* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/go-fsnotify/fsnotify/issues/36) + +## v1.0.2 / 2014-08-17 + +* [Fix] Missing create events on OS X. [#14](https://github.com/go-fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Fix] Make ./path and path equivalent. (thanks @zhsso) + +## v1.0.0 / 2014-08-15 + +* [API] Remove AddWatch on Windows, use Add. +* Improve documentation for exported identifiers. [#30](https://github.com/go-fsnotify/fsnotify/issues/30) +* Minor updates based on feedback from golint. + +## dev / 2014-07-09 + +* Moved to [github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify). +* Use os.NewSyscallError instead of returning errno (thanks @hariharan-uno) + +## dev / 2014-07-04 + +* kqueue: fix incorrect mutex used in Close() +* Update example to demonstrate usage of Op. + +## dev / 2014-06-28 + +* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/go-fsnotify/fsnotify/issues/4) +* Fix for String() method on Event (thanks Alex Brainman) +* Don't build on Plan 9 or Solaris (thanks @4ad) + +## dev / 2014-06-21 + +* Events channel of type Event rather than *Event. +* [internal] use syscall constants directly for inotify and kqueue. +* [internal] kqueue: rename events to kevents and fileEvent to event. + +## dev / 2014-06-19 + +* Go 1.3+ required on Windows (uses syscall.ERROR_MORE_DATA internally). +* [internal] remove cookie from Event struct (unused). +* [internal] Event struct has the same definition across every OS. +* [internal] remove internal watch and removeWatch methods. + +## dev / 2014-06-12 + +* [API] Renamed Watch() to Add() and RemoveWatch() to Remove(). +* [API] Pluralized channel names: Events and Errors. +* [API] Renamed FileEvent struct to Event. +* [API] Op constants replace methods like IsCreate(). + +## dev / 2014-06-12 + +* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) + +## dev / 2014-05-23 + +* [API] Remove current implementation of WatchFlags. + * current implementation doesn't take advantage of OS for efficiency + * provides little benefit over filtering events as they are received, but has extra bookkeeping and mutexes + * no tests for the current implementation + * not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195) + +## v0.9.3 / 2014-12-31 + +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/go-fsnotify/fsnotify/issues/51) + +## v0.9.2 / 2014-08-17 + +* [Backport] Fix missing create events on OS X. [#14](https://github.com/go-fsnotify/fsnotify/issues/14) (thanks @zhsso) + +## v0.9.1 / 2014-06-12 + +* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) + +## v0.9.0 / 2014-01-17 + +* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) +* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) +* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. + +## v0.8.12 / 2013-11-13 + +* [API] Remove FD_SET and friends from Linux adapter + +## v0.8.11 / 2013-11-02 + +* [Doc] Add Changelog [#72][] (thanks @nathany) +* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond) + +## v0.8.10 / 2013-10-19 + +* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) +* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) +* [Doc] specify OS-specific limits in README (thanks @debrando) + +## v0.8.9 / 2013-09-08 + +* [Doc] Contributing (thanks @nathany) +* [Doc] update package path in example code [#63][] (thanks @paulhammond) +* [Doc] GoCI badge in README (Linux only) [#60][] +* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) + +## v0.8.8 / 2013-06-17 + +* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) + +## v0.8.7 / 2013-06-03 + +* [API] Make syscall flags internal +* [Fix] inotify: ignore event changes +* [Fix] race in symlink test [#45][] (reported by @srid) +* [Fix] tests on Windows +* lower case error messages + +## v0.8.6 / 2013-05-23 + +* kqueue: Use EVT_ONLY flag on Darwin +* [Doc] Update README with full example + +## v0.8.5 / 2013-05-09 + +* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) + +## v0.8.4 / 2013-04-07 + +* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) + +## v0.8.3 / 2013-03-13 + +* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) +* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) + +## v0.8.2 / 2013-02-07 + +* [Doc] add Authors +* [Fix] fix data races for map access [#29][] (thanks @fsouza) + +## v0.8.1 / 2013-01-09 + +* [Fix] Windows path separators +* [Doc] BSD License + +## v0.8.0 / 2012-11-09 + +* kqueue: directory watching improvements (thanks @vmirage) +* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) +* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) + +## v0.7.4 / 2012-10-09 + +* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) +* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) +* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) +* [Fix] kqueue: modify after recreation of file + +## v0.7.3 / 2012-09-27 + +* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) +* [Fix] kqueue: no longer get duplicate CREATE events + +## v0.7.2 / 2012-09-01 + +* kqueue: events for created directories + +## v0.7.1 / 2012-07-14 + +* [Fix] for renaming files + +## v0.7.0 / 2012-07-02 + +* [Feature] FSNotify flags +* [Fix] inotify: Added file name back to event path + +## v0.6.0 / 2012-06-06 + +* kqueue: watch files after directory created (thanks @tmc) + +## v0.5.1 / 2012-05-22 + +* [Fix] inotify: remove all watches before Close() + +## v0.5.0 / 2012-05-03 + +* [API] kqueue: return errors during watch instead of sending over channel +* kqueue: match symlink behavior on Linux +* inotify: add `DELETE_SELF` (requested by @taralx) +* [Fix] kqueue: handle EINTR (reported by @robfig) +* [Doc] Godoc example [#1][] (thanks @davecheney) + +## v0.4.0 / 2012-03-30 + +* Go 1 released: build with go tool +* [Feature] Windows support using winfsnotify +* Windows does not have attribute change notifications +* Roll attribute notifications into IsModify + +## v0.3.0 / 2012-02-19 + +* kqueue: add files when watch directory + +## v0.2.0 / 2011-12-30 + +* update to latest Go weekly code + +## v0.1.0 / 2011-10-19 + +* kqueue: add watch on file creation to match inotify +* kqueue: create file event +* inotify: ignore `IN_IGNORED` events +* event String() +* linux: common FileEvent functions +* initial commit + +[#79]: https://github.com/howeyc/fsnotify/pull/79 +[#77]: https://github.com/howeyc/fsnotify/pull/77 +[#72]: https://github.com/howeyc/fsnotify/issues/72 +[#71]: https://github.com/howeyc/fsnotify/issues/71 +[#70]: https://github.com/howeyc/fsnotify/issues/70 +[#63]: https://github.com/howeyc/fsnotify/issues/63 +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#60]: https://github.com/howeyc/fsnotify/issues/60 +[#59]: https://github.com/howeyc/fsnotify/issues/59 +[#49]: https://github.com/howeyc/fsnotify/issues/49 +[#45]: https://github.com/howeyc/fsnotify/issues/45 +[#40]: https://github.com/howeyc/fsnotify/issues/40 +[#36]: https://github.com/howeyc/fsnotify/issues/36 +[#33]: https://github.com/howeyc/fsnotify/issues/33 +[#29]: https://github.com/howeyc/fsnotify/issues/29 +[#25]: https://github.com/howeyc/fsnotify/issues/25 +[#24]: https://github.com/howeyc/fsnotify/issues/24 +[#21]: https://github.com/howeyc/fsnotify/issues/21 + diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CONTRIBUTING.md b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CONTRIBUTING.md new file mode 100644 index 0000000000000..0f377f341b0c1 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/CONTRIBUTING.md @@ -0,0 +1,77 @@ +# Contributing + +## Issues + +* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/go-fsnotify/fsnotify/issues). +* Please indicate the platform you are using fsnotify on. +* A code example to reproduce the problem is appreciated. + +## Pull Requests + +### Contributor License Agreement + +fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/go-fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/go-fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). + +Please indicate that you have signed the CLA in your pull request. + +### How fsnotify is Developed + +* Development is done on feature branches. +* Tests are run on BSD, Linux, OS X and Windows. +* Pull requests are reviewed and [applied to master][am] using [hub][]. + * Maintainers may modify or squash commits rather than asking contributors to. +* To issue a new release, the maintainers will: + * Update the CHANGELOG + * Tag a version, which will become available through gopkg.in. + +### How to Fork + +For smooth sailing, always use the original import path. Installing with `go get` makes this easy. + +1. Install from GitHub (`go get -u github.com/go-fsnotify/fsnotify`) +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Ensure everything works and the tests pass (see below) +4. Commit your changes (`git commit -am 'Add some feature'`) + +Contribute upstream: + +1. Fork fsnotify on GitHub +2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) +3. Push to the branch (`git push fork my-new-feature`) +4. Create a new Pull Request on GitHub + +This workflow is [thoroughly explained by Katrina Owen](https://blog.splice.com/contributing-open-source-git-repositories-go/). + +### Testing + +fsnotify uses build tags to compile different code on Linux, BSD, OS X, and Windows. + +Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. + +To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. + +* Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/) +* Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder. +* Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password) +* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd go-fsnotify/fsnotify; go test'`. +* When you're done, you will want to halt or destroy the Vagrant boxes. + +Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. + +Right now there is no equivalent solution for Windows and OS X, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). + +### Maintainers + +Help maintaining fsnotify is welcome. To be a maintainer: + +* Submit a pull request and sign the CLA as above. +* You must be able to run the test suite on Mac, Windows, Linux and BSD. + +To keep master clean, the fsnotify project uses the "apply mail" workflow outlined in Nathaniel Talbott's post ["Merge pull request" Considered Harmful][am]. This requires installing [hub][]. + +All code changes should be internal pull requests. + +Releases are tagged using [Semantic Versioning](http://semver.org/). + +[hub]: https://github.com/github/hub +[am]: http://blog.spreedly.com/2014/06/24/merge-pull-request-considered-harmful/#.VGa5yZPF_Zs diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/LICENSE b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/LICENSE new file mode 100644 index 0000000000000..f21e54080090f --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md new file mode 100644 index 0000000000000..736c0d27a6c71 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/README.md @@ -0,0 +1,64 @@ +# File system notifications for Go + +[![GoDoc](https://godoc.org/gopkg.in/fsnotify.v1?status.svg)](https://godoc.org/gopkg.in/fsnotify.v1) [![Coverage](http://gocover.io/_badge/github.com/go-fsnotify/fsnotify)](http://gocover.io/github.com/go-fsnotify/fsnotify) + +Go 1.3+ required. + +Cross platform: Windows, Linux, BSD and OS X. + +|Adapter |OS |Status | +|----------|----------|----------| +|inotify |Linux, Android\*|Supported [![Build Status](https://travis-ci.org/go-fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/go-fsnotify/fsnotify)| +|kqueue |BSD, OS X, iOS\*|Supported [![Build Status](https://travis-ci.org/go-fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/go-fsnotify/fsnotify)| +|ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| +|FSEvents |OS X |[Planned](https://github.com/go-fsnotify/fsnotify/issues/11)| +|FEN |Solaris 11 |[Planned](https://github.com/go-fsnotify/fsnotify/issues/12)| +|fanotify |Linux 2.6.37+ | | +|USN Journals |Windows |[Maybe](https://github.com/go-fsnotify/fsnotify/issues/53)| +|Polling |*All* |[Maybe](https://github.com/go-fsnotify/fsnotify/issues/9)| + +\* Android and iOS are untested. + +Please see [the documentation](https://godoc.org/gopkg.in/fsnotify.v1) for usage. Consult the [Wiki](https://github.com/go-fsnotify/fsnotify/wiki) for the FAQ and further information. + +## API stability + +Two major versions of fsnotify exist. + +**[fsnotify.v0](https://gopkg.in/fsnotify.v0)** is API-compatible with [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify). Bugfixes *may* be backported, but I recommend upgrading to v1. + +```go +import "gopkg.in/fsnotify.v0" +``` + +\* Refer to the package as fsnotify (without the .v0 suffix). + +**[fsnotify.v1](https://gopkg.in/fsnotify.v1)** provides [a new API](https://godoc.org/gopkg.in/fsnotify.v1) based on [this design document](http://goo.gl/MrYxyA). You can import v1 with: + +```go +import "gopkg.in/fsnotify.v1" +``` + +Further API changes are [planned](https://github.com/go-fsnotify/fsnotify/milestones), but a new major revision will be tagged, so you can depend on the v1 API. + +**Master** may have unreleased changes. Use it to test the very latest code or when [contributing][], but don't expect it to remain API-compatible: + +```go +import "github.com/go-fsnotify/fsnotify" +``` + +## Contributing + +Please refer to [CONTRIBUTING][] before opening an issue or pull request. + +## Example + +See [example_test.go](https://github.com/go-fsnotify/fsnotify/blob/master/example_test.go). + +[contributing]: https://github.com/go-fsnotify/fsnotify/blob/master/CONTRIBUTING.md + +## Related Projects + +* [notify](https://github.com/rjeczalik/notify) +* [fsevents](https://github.com/go-fsnotify/fsevents) + diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/example_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/example_test.go new file mode 100644 index 0000000000000..3063796602ce4 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/example_test.go @@ -0,0 +1,42 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!solaris + +package fsnotify_test + +import ( + "log" + + "github.com/go-fsnotify/fsnotify" +) + +func ExampleNewWatcher() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + done := make(chan bool) + go func() { + for { + select { + case event := <-watcher.Events: + log.Println("event:", event) + if event.Op&fsnotify.Write == fsnotify.Write { + log.Println("modified file:", event.Name) + } + case err := <-watcher.Errors: + log.Println("error:", err) + } + } + }() + + err = watcher.Add("/tmp/foo") + if err != nil { + log.Fatal(err) + } + <-done +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/fsnotify.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/fsnotify.go new file mode 100644 index 0000000000000..c899ee0083877 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/fsnotify.go @@ -0,0 +1,62 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!solaris + +// Package fsnotify provides a platform-independent interface for file system notifications. +package fsnotify + +import ( + "bytes" + "fmt" +) + +// Event represents a single file system notification. +type Event struct { + Name string // Relative path to the file or directory. + Op Op // File operation that triggered the event. +} + +// Op describes a set of file operations. +type Op uint32 + +// These are the generalized file operations that can trigger a notification. +const ( + Create Op = 1 << iota + Write + Remove + Rename + Chmod +) + +// String returns a string representation of the event in the form +// "file: REMOVE|WRITE|..." +func (e Event) String() string { + // Use a buffer for efficient string concatenation + var buffer bytes.Buffer + + if e.Op&Create == Create { + buffer.WriteString("|CREATE") + } + if e.Op&Remove == Remove { + buffer.WriteString("|REMOVE") + } + if e.Op&Write == Write { + buffer.WriteString("|WRITE") + } + if e.Op&Rename == Rename { + buffer.WriteString("|RENAME") + } + if e.Op&Chmod == Chmod { + buffer.WriteString("|CHMOD") + } + + // If buffer remains empty, return no event names + if buffer.Len() == 0 { + return fmt.Sprintf("%q: ", e.Name) + } + + // Return a list of event names, with leading pipe character stripped + return fmt.Sprintf("%q: %s", e.Name, buffer.String()[1:]) +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go new file mode 100644 index 0000000000000..d7759ec8c86b0 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify.go @@ -0,0 +1,306 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "unsafe" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + mu sync.Mutex // Map access + fd int + poller *fdPoller + watches map[string]*watch // Map of inotify watches (key: path) + paths map[int]string // Map of watched paths (key: watch descriptor) + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + doneResp chan struct{} // Channel to respond to Close +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + // Create inotify fd + fd, errno := syscall.InotifyInit() + if fd == -1 { + return nil, errno + } + // Create epoll + poller, err := newFdPoller(fd) + if err != nil { + syscall.Close(fd) + return nil, err + } + w := &Watcher{ + fd: fd, + poller: poller, + watches: make(map[string]*watch), + paths: make(map[int]string), + Events: make(chan Event), + Errors: make(chan error), + done: make(chan struct{}), + doneResp: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +func (w *Watcher) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + if w.isClosed() { + return nil + } + + // Send 'close' signal to goroutine, and set the Watcher to closed. + close(w.done) + + // Wake up goroutine + w.poller.wake() + + // Wait for goroutine to close + <-w.doneResp + + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + name = filepath.Clean(name) + if w.isClosed() { + return errors.New("inotify instance already closed") + } + + const agnosticEvents = syscall.IN_MOVED_TO | syscall.IN_MOVED_FROM | + syscall.IN_CREATE | syscall.IN_ATTRIB | syscall.IN_MODIFY | + syscall.IN_MOVE_SELF | syscall.IN_DELETE | syscall.IN_DELETE_SELF + + var flags uint32 = agnosticEvents + + w.mu.Lock() + watchEntry, found := w.watches[name] + w.mu.Unlock() + if found { + watchEntry.flags |= flags + flags |= syscall.IN_MASK_ADD + } + wd, errno := syscall.InotifyAddWatch(w.fd, name, flags) + if wd == -1 { + return errno + } + + w.mu.Lock() + w.watches[name] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = name + w.mu.Unlock() + + return nil +} + +// Remove stops watching the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + name = filepath.Clean(name) + + // Fetch the watch. + w.mu.Lock() + defer w.mu.Unlock() + watch, ok := w.watches[name] + + // Remove it from inotify. + if !ok { + return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) + } + // inotify_rm_watch will return EINVAL if the file has been deleted; + // the inotify will already have been removed. + // That means we can safely delete it from our watches, whatever inotify_rm_watch does. + delete(w.watches, name) + success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) + if success == -1 { + // TODO: Perhaps it's not helpful to return an error here in every case. + // the only two possible errors are: + // EBADF, which happens when w.fd is not a valid file descriptor of any kind. + // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. + // Watch descriptors are invalidated when they are removed explicitly or implicitly; + // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. + return errno + } + return nil +} + +type watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Events channel +func (w *Watcher) readEvents() { + var ( + buf [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + n int // Number of bytes read with read() + errno error // Syscall errno + ok bool // For poller.wait + ) + + defer close(w.doneResp) + defer close(w.Errors) + defer close(w.Events) + defer syscall.Close(w.fd) + defer w.poller.close() + + for { + // See if we have been closed. + if w.isClosed() { + return + } + + ok, errno = w.poller.wait() + if errno != nil { + select { + case w.Errors <- errno: + case <-w.done: + return + } + continue + } + + if !ok { + continue + } + + n, errno = syscall.Read(w.fd, buf[:]) + // If a signal interrupted execution, see if we've been asked to close, and try again. + // http://man7.org/linux/man-pages/man7/signal.7.html : + // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" + if errno == syscall.EINTR { + continue + } + + // syscall.Read might have been woken up by Close. If so, we're done. + if w.isClosed() { + return + } + + if n < syscall.SizeofInotifyEvent { + var err error + if n == 0 { + // If EOF is received. This should really never happen. + err = io.EOF + } else if n < 0 { + // If an error occured while reading. + err = errno + } else { + // Read was too short. + err = errors.New("notify: short read in readEvents()") + } + select { + case w.Errors <- err: + case <-w.done: + return + } + continue + } + + var offset uint32 + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-syscall.SizeofInotifyEvent) { + // Point "raw" to the event in the buffer + raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset])) + + mask := uint32(raw.Mask) + nameLen := uint32(raw.Len) + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + w.mu.Lock() + name := w.paths[int(raw.Wd)] + w.mu.Unlock() + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent])) + // The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + event := newEvent(name, mask) + + // Send the events that are not ignored on the events channel + if !event.ignoreLinux(mask) { + select { + case w.Events <- event: + case <-w.done: + return + } + } + + // Move to the next event in the buffer + offset += syscall.SizeofInotifyEvent + nameLen + } + } +} + +// Certain types of events can be "ignored" and not sent over the Events +// channel. Such as events marked ignore by the kernel, or MODIFY events +// against files that do not exist. +func (e *Event) ignoreLinux(mask uint32) bool { + // Ignore anything the inotify API says to ignore + if mask&syscall.IN_IGNORED == syscall.IN_IGNORED { + return true + } + + // If the event is not a DELETE or RENAME, the file must exist. + // Otherwise the event is ignored. + // *Note*: this was put in place because it was seen that a MODIFY + // event was sent after the DELETE. This ignores that MODIFY and + // assumes a DELETE will come or has come if the file doesn't exist. + if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { + _, statErr := os.Lstat(e.Name) + return os.IsNotExist(statErr) + } + return false +} + +// newEvent returns an platform-independent Event based on an inotify mask. +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&syscall.IN_CREATE == syscall.IN_CREATE || mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO { + e.Op |= Create + } + if mask&syscall.IN_DELETE_SELF == syscall.IN_DELETE_SELF || mask&syscall.IN_DELETE == syscall.IN_DELETE { + e.Op |= Remove + } + if mask&syscall.IN_MODIFY == syscall.IN_MODIFY { + e.Op |= Write + } + if mask&syscall.IN_MOVE_SELF == syscall.IN_MOVE_SELF || mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM { + e.Op |= Rename + } + if mask&syscall.IN_ATTRIB == syscall.IN_ATTRIB { + e.Op |= Chmod + } + return e +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go new file mode 100644 index 0000000000000..3b4178404196e --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller.go @@ -0,0 +1,186 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "errors" + "syscall" +) + +type fdPoller struct { + fd int // File descriptor (as returned by the inotify_init() syscall) + epfd int // Epoll file descriptor + pipe [2]int // Pipe for waking up +} + +func emptyPoller(fd int) *fdPoller { + poller := new(fdPoller) + poller.fd = fd + poller.epfd = -1 + poller.pipe[0] = -1 + poller.pipe[1] = -1 + return poller +} + +// Create a new inotify poller. +// This creates an inotify handler, and an epoll handler. +func newFdPoller(fd int) (*fdPoller, error) { + var errno error + poller := emptyPoller(fd) + defer func() { + if errno != nil { + poller.close() + } + }() + poller.fd = fd + + // Create epoll fd + poller.epfd, errno = syscall.EpollCreate(1) + if poller.epfd == -1 { + return nil, errno + } + // Create pipe; pipe[0] is the read end, pipe[1] the write end. + errno = syscall.Pipe2(poller.pipe[:], syscall.O_NONBLOCK) + if errno != nil { + return nil, errno + } + + // Register inotify fd with epoll + event := syscall.EpollEvent{ + Fd: int32(poller.fd), + Events: syscall.EPOLLIN, + } + errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.fd, &event) + if errno != nil { + return nil, errno + } + + // Register pipe fd with epoll + event = syscall.EpollEvent{ + Fd: int32(poller.pipe[0]), + Events: syscall.EPOLLIN, + } + errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.pipe[0], &event) + if errno != nil { + return nil, errno + } + + return poller, nil +} + +// Wait using epoll. +// Returns true if something is ready to be read, +// false if there is not. +func (poller *fdPoller) wait() (bool, error) { + // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. + // I don't know whether epoll_wait returns the number of events returned, + // or the total number of events ready. + // I decided to catch both by making the buffer one larger than the maximum. + events := make([]syscall.EpollEvent, 7) + for { + n, errno := syscall.EpollWait(poller.epfd, events, -1) + if n == -1 { + if errno == syscall.EINTR { + continue + } + return false, errno + } + if n == 0 { + // If there are no events, try again. + continue + } + if n > 6 { + // This should never happen. More events were returned than should be possible. + return false, errors.New("epoll_wait returned more events than I know what to do with") + } + ready := events[:n] + epollhup := false + epollerr := false + epollin := false + for _, event := range ready { + if event.Fd == int32(poller.fd) { + if event.Events&syscall.EPOLLHUP != 0 { + // This should not happen, but if it does, treat it as a wakeup. + epollhup = true + } + if event.Events&syscall.EPOLLERR != 0 { + // If an error is waiting on the file descriptor, we should pretend + // something is ready to read, and let syscall.Read pick up the error. + epollerr = true + } + if event.Events&syscall.EPOLLIN != 0 { + // There is data to read. + epollin = true + } + } + if event.Fd == int32(poller.pipe[0]) { + if event.Events&syscall.EPOLLHUP != 0 { + // Write pipe descriptor was closed, by us. This means we're closing down the + // watcher, and we should wake up. + } + if event.Events&syscall.EPOLLERR != 0 { + // If an error is waiting on the pipe file descriptor. + // This is an absolute mystery, and should never ever happen. + return false, errors.New("Error on the pipe descriptor.") + } + if event.Events&syscall.EPOLLIN != 0 { + // This is a regular wakeup, so we have to clear the buffer. + err := poller.clearWake() + if err != nil { + return false, err + } + } + } + } + + if epollhup || epollerr || epollin { + return true, nil + } + return false, nil + } +} + +// Close the write end of the poller. +func (poller *fdPoller) wake() error { + buf := make([]byte, 1) + n, errno := syscall.Write(poller.pipe[1], buf) + if n == -1 { + if errno == syscall.EAGAIN { + // Buffer is full, poller will wake. + return nil + } + return errno + } + return nil +} + +func (poller *fdPoller) clearWake() error { + // You have to be woken up a LOT in order to get to 100! + buf := make([]byte, 100) + n, errno := syscall.Read(poller.pipe[0], buf) + if n == -1 { + if errno == syscall.EAGAIN { + // Buffer is empty, someone else cleared our wake. + return nil + } + return errno + } + return nil +} + +// Close all poller file descriptors, but not the one passed to it. +func (poller *fdPoller) close() { + if poller.pipe[1] != -1 { + syscall.Close(poller.pipe[1]) + } + if poller.pipe[0] != -1 { + syscall.Close(poller.pipe[0]) + } + if poller.epfd != -1 { + syscall.Close(poller.epfd) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller_test.go new file mode 100644 index 0000000000000..af9f407f8d928 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_poller_test.go @@ -0,0 +1,228 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "syscall" + "testing" + "time" +) + +type testFd [2]int + +func makeTestFd(t *testing.T) testFd { + var tfd testFd + errno := syscall.Pipe(tfd[:]) + if errno != nil { + t.Fatalf("Failed to create pipe: %v", errno) + } + return tfd +} + +func (tfd testFd) fd() int { + return tfd[0] +} + +func (tfd testFd) closeWrite(t *testing.T) { + errno := syscall.Close(tfd[1]) + if errno != nil { + t.Fatalf("Failed to close write end of pipe: %v", errno) + } +} + +func (tfd testFd) put(t *testing.T) { + buf := make([]byte, 10) + _, errno := syscall.Write(tfd[1], buf) + if errno != nil { + t.Fatalf("Failed to write to pipe: %v", errno) + } +} + +func (tfd testFd) get(t *testing.T) { + buf := make([]byte, 10) + _, errno := syscall.Read(tfd[0], buf) + if errno != nil { + t.Fatalf("Failed to read from pipe: %v", errno) + } +} + +func (tfd testFd) close() { + syscall.Close(tfd[1]) + syscall.Close(tfd[0]) +} + +func makePoller(t *testing.T) (testFd, *fdPoller) { + tfd := makeTestFd(t) + poller, err := newFdPoller(tfd.fd()) + if err != nil { + t.Fatalf("Failed to create poller: %v", err) + } + return tfd, poller +} + +func TestPollerWithBadFd(t *testing.T) { + _, err := newFdPoller(-1) + if err != syscall.EBADF { + t.Fatalf("Expected EBADF, got: %v", err) + } +} + +func TestPollerWithData(t *testing.T) { + tfd, poller := makePoller(t) + defer tfd.close() + defer poller.close() + + tfd.put(t) + ok, err := poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if !ok { + t.Fatalf("expected poller to return true") + } + tfd.get(t) +} + +func TestPollerWithWakeup(t *testing.T) { + tfd, poller := makePoller(t) + defer tfd.close() + defer poller.close() + + err := poller.wake() + if err != nil { + t.Fatalf("wake failed: %v", err) + } + ok, err := poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if ok { + t.Fatalf("expected poller to return false") + } +} + +func TestPollerWithClose(t *testing.T) { + tfd, poller := makePoller(t) + defer tfd.close() + defer poller.close() + + tfd.closeWrite(t) + ok, err := poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if !ok { + t.Fatalf("expected poller to return true") + } +} + +func TestPollerWithWakeupAndData(t *testing.T) { + tfd, poller := makePoller(t) + defer tfd.close() + defer poller.close() + + tfd.put(t) + err := poller.wake() + if err != nil { + t.Fatalf("wake failed: %v", err) + } + + // both data and wakeup + ok, err := poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if !ok { + t.Fatalf("expected poller to return true") + } + + // data is still in the buffer, wakeup is cleared + ok, err = poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if !ok { + t.Fatalf("expected poller to return true") + } + + tfd.get(t) + // data is gone, only wakeup now + err = poller.wake() + if err != nil { + t.Fatalf("wake failed: %v", err) + } + ok, err = poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + if ok { + t.Fatalf("expected poller to return false") + } +} + +func TestPollerConcurrent(t *testing.T) { + tfd, poller := makePoller(t) + defer tfd.close() + defer poller.close() + + oks := make(chan bool) + live := make(chan bool) + defer close(live) + go func() { + defer close(oks) + for { + ok, err := poller.wait() + if err != nil { + t.Fatalf("poller failed: %v", err) + } + oks <- ok + if !<-live { + return + } + } + }() + + // Try a write + select { + case <-time.After(50 * time.Millisecond): + case <-oks: + t.Fatalf("poller did not wait") + } + tfd.put(t) + if !<-oks { + t.Fatalf("expected true") + } + tfd.get(t) + live <- true + + // Try a wakeup + select { + case <-time.After(50 * time.Millisecond): + case <-oks: + t.Fatalf("poller did not wait") + } + err := poller.wake() + if err != nil { + t.Fatalf("wake failed: %v", err) + } + if <-oks { + t.Fatalf("expected false") + } + live <- true + + // Try a close + select { + case <-time.After(50 * time.Millisecond): + case <-oks: + t.Fatalf("poller did not wait") + } + tfd.closeWrite(t) + if !<-oks { + t.Fatalf("expected true") + } + tfd.get(t) +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go new file mode 100644 index 0000000000000..035ee8f95d599 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/inotify_test.go @@ -0,0 +1,292 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestInotifyCloseRightAway(t *testing.T) { + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher") + } + + // Close immediately; it won't even reach the first syscall.Read. + w.Close() + + // Wait for the close to complete. + <-time.After(50 * time.Millisecond) + isWatcherReallyClosed(t, w) +} + +func TestInotifyCloseSlightlyLater(t *testing.T) { + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher") + } + + // Wait until readEvents has reached syscall.Read, and Close. + <-time.After(50 * time.Millisecond) + w.Close() + + // Wait for the close to complete. + <-time.After(50 * time.Millisecond) + isWatcherReallyClosed(t, w) +} + +func TestInotifyCloseSlightlyLaterWithWatch(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher") + } + w.Add(testDir) + + // Wait until readEvents has reached syscall.Read, and Close. + <-time.After(50 * time.Millisecond) + w.Close() + + // Wait for the close to complete. + <-time.After(50 * time.Millisecond) + isWatcherReallyClosed(t, w) +} + +func TestInotifyCloseAfterRead(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher") + } + + err = w.Add(testDir) + if err != nil { + t.Fatalf("Failed to add .") + } + + // Generate an event. + os.Create(filepath.Join(testDir, "somethingSOMETHINGsomethingSOMETHING")) + + // Wait for readEvents to read the event, then close the watcher. + <-time.After(50 * time.Millisecond) + w.Close() + + // Wait for the close to complete. + <-time.After(50 * time.Millisecond) + isWatcherReallyClosed(t, w) +} + +func isWatcherReallyClosed(t *testing.T, w *Watcher) { + select { + case err, ok := <-w.Errors: + if ok { + t.Fatalf("w.Errors is not closed; readEvents is still alive after closing (error: %v)", err) + } + default: + t.Fatalf("w.Errors would have blocked; readEvents is still alive!") + } + + select { + case _, ok := <-w.Events: + if ok { + t.Fatalf("w.Events is not closed; readEvents is still alive after closing") + } + default: + t.Fatalf("w.Events would have blocked; readEvents is still alive!") + } +} + +func TestInotifyCloseCreate(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher: %v", err) + } + defer w.Close() + + err = w.Add(testDir) + if err != nil { + t.Fatalf("Failed to add testDir: %v", err) + } + h, err := os.Create(filepath.Join(testDir, "testfile")) + if err != nil { + t.Fatalf("Failed to create file in testdir: %v", err) + } + h.Close() + select { + case _ = <-w.Events: + case err := <-w.Errors: + t.Fatalf("Error from watcher: %v", err) + case <-time.After(50 * time.Millisecond): + t.Fatalf("Took too long to wait for event") + } + + // At this point, we've received one event, so the goroutine is ready. + // It's also blocking on syscall.Read. + // Now we try to swap the file descriptor under its nose. + w.Close() + w, err = NewWatcher() + defer w.Close() + if err != nil { + t.Fatalf("Failed to create second watcher: %v", err) + } + + <-time.After(50 * time.Millisecond) + err = w.Add(testDir) + if err != nil { + t.Fatalf("Error adding testDir again: %v", err) + } +} + +func TestInotifyStress(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + testFile := filepath.Join(testDir, "testfile") + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher: %v", err) + } + defer w.Close() + + killchan := make(chan struct{}) + defer close(killchan) + + err = w.Add(testDir) + if err != nil { + t.Fatalf("Failed to add testDir: %v", err) + } + + proc, err := os.FindProcess(os.Getpid()) + if err != nil { + t.Fatalf("Error finding process: %v", err) + } + + go func() { + for { + select { + case <-time.After(5 * time.Millisecond): + err := proc.Signal(syscall.SIGUSR1) + if err != nil { + t.Fatalf("Signal failed: %v", err) + } + case <-killchan: + return + } + } + }() + + go func() { + for { + select { + case <-time.After(11 * time.Millisecond): + err := w.poller.wake() + if err != nil { + t.Fatalf("Wake failed: %v", err) + } + case <-killchan: + return + } + } + }() + + go func() { + for { + select { + case <-killchan: + return + default: + handle, err := os.Create(testFile) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + handle.Close() + time.Sleep(time.Millisecond) + err = os.Remove(testFile) + if err != nil { + t.Fatalf("Remove failed: %v", err) + } + } + } + }() + + creates := 0 + removes := 0 + after := time.After(5 * time.Second) + for { + select { + case <-after: + if creates-removes > 1 || creates-removes < -1 { + t.Fatalf("Creates and removes should not be off by more than one: %d creates, %d removes", creates, removes) + } + if creates < 50 { + t.Fatalf("Expected at least 50 creates, got %d", creates) + } + return + case err := <-w.Errors: + t.Fatalf("Got an error from watcher: %v", err) + case evt := <-w.Events: + if evt.Name != testFile { + t.Fatalf("Got an event for an unknown file: %s", evt.Name) + } + if evt.Op == Create { + creates++ + } + if evt.Op == Remove { + removes++ + } + } + } +} + +func TestInotifyRemoveTwice(t *testing.T) { + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + testFile := filepath.Join(testDir, "testfile") + + handle, err := os.Create(testFile) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + handle.Close() + + w, err := NewWatcher() + if err != nil { + t.Fatalf("Failed to create watcher: %v", err) + } + defer w.Close() + + err = w.Add(testFile) + if err != nil { + t.Fatalf("Failed to add testFile: %v", err) + } + + err = os.Remove(testFile) + if err != nil { + t.Fatalf("Failed to remove testFile: %v", err) + } + + err = w.Remove(testFile) + if err != syscall.EINVAL { + t.Fatalf("Expected EINVAL from Remove, got: %v", err) + } + + err = w.Remove(testFile) + if err == syscall.EINVAL { + t.Fatalf("Got EINVAL again, watch was not removed") + } +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go new file mode 100644 index 0000000000000..59169c6afa881 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/integration_test.go @@ -0,0 +1,1135 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!solaris + +package fsnotify + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" +) + +// An atomic counter +type counter struct { + val int32 +} + +func (c *counter) increment() { + atomic.AddInt32(&c.val, 1) +} + +func (c *counter) value() int32 { + return atomic.LoadInt32(&c.val) +} + +func (c *counter) reset() { + atomic.StoreInt32(&c.val, 0) +} + +// tempMkdir makes a temporary directory +func tempMkdir(t *testing.T) string { + dir, err := ioutil.TempDir("", "fsnotify") + if err != nil { + t.Fatalf("failed to create test directory: %s", err) + } + return dir +} + +// newWatcher initializes an fsnotify Watcher instance. +func newWatcher(t *testing.T) *Watcher { + watcher, err := NewWatcher() + if err != nil { + t.Fatalf("NewWatcher() failed: %s", err) + } + return watcher +} + +// addWatch adds a watch for a directory +func addWatch(t *testing.T, watcher *Watcher, dir string) { + if err := watcher.Add(dir); err != nil { + t.Fatalf("watcher.Add(%q) failed: %s", dir, err) + } +} + +func TestFsnotifyMultipleOperations(t *testing.T) { + watcher := newWatcher(t) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory that's not watched + testDirToMoveFiles := tempMkdir(t) + defer os.RemoveAll(testDirToMoveFiles) + + testFile := filepath.Join(testDir, "TestFsnotifySeq.testfile") + testFileRenamed := filepath.Join(testDirToMoveFiles, "TestFsnotifySeqRename.testfile") + + addWatch(t, watcher, testDir) + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var createReceived, modifyReceived, deleteReceived, renameReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + t.Logf("event received: %s", event) + if event.Op&Remove == Remove { + deleteReceived.increment() + } + if event.Op&Write == Write { + modifyReceived.increment() + } + if event.Op&Create == Create { + createReceived.increment() + } + if event.Op&Rename == Rename { + renameReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // Modify the file outside of the watched dir + f, err = os.Open(testFileRenamed) + if err != nil { + t.Fatalf("open test renamed file failed: %s", err) + } + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Recreate the file that was moved + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Close() + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + mReceived := modifyReceived.value() + if mReceived != 1 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs %d)", mReceived, 1) + } + dReceived := deleteReceived.value() + rReceived := renameReceived.value() + if dReceived+rReceived != 1 { + t.Fatalf("incorrect number of rename+delete events received after 500 ms (%d vs %d)", rReceived+dReceived, 1) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyMultipleCreates(t *testing.T) { + watcher := newWatcher(t) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + testFile := filepath.Join(testDir, "TestFsnotifySeq.testfile") + + addWatch(t, watcher, testDir) + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var createReceived, modifyReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + t.Logf("event received: %s", event) + if event.Op&Remove == Remove { + deleteReceived.increment() + } + if event.Op&Create == Create { + createReceived.increment() + } + if event.Op&Write == Write { + modifyReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + os.Remove(testFile) + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Recreate the file + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Close() + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Modify + f, err = os.OpenFile(testFile, os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // Modify + f, err = os.OpenFile(testFile, os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + mReceived := modifyReceived.value() + if mReceived < 3 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs atleast %d)", mReceived, 3) + } + dReceived := deleteReceived.value() + if dReceived != 1 { + t.Fatalf("incorrect number of rename+delete events received after 500 ms (%d vs %d)", dReceived, 1) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyDirOnly(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + // This should NOT add any events to the fsnotify event queue + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyDirOnly.testfile") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var createReceived, modifyReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileAlreadyExists) { + t.Logf("event received: %s", event) + if event.Op&Remove == Remove { + deleteReceived.increment() + } + if event.Op&Write == Write { + modifyReceived.increment() + } + if event.Op&Create == Create { + createReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + time.Sleep(time.Millisecond) + f.WriteString("data") + f.Sync() + f.Close() + + time.Sleep(50 * time.Millisecond) // give system time to sync write change before delete + + os.Remove(testFile) + os.Remove(testFileAlreadyExists) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 1 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 1) + } + mReceived := modifyReceived.value() + if mReceived != 1 { + t.Fatalf("incorrect number of modify events received after 500 ms (%d vs %d)", mReceived, 1) + } + dReceived := deleteReceived.value() + if dReceived != 2 { + t.Fatalf("incorrect number of delete events received after 500 ms (%d vs %d)", dReceived, 2) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyDeleteWatchedDir(t *testing.T) { + watcher := newWatcher(t) + defer watcher.Close() + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + addWatch(t, watcher, testDir) + + // Add a watch for testFile + addWatch(t, watcher, testFileAlreadyExists) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var deleteReceived counter + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFileAlreadyExists) { + t.Logf("event received: %s", event) + if event.Op&Remove == Remove { + deleteReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + }() + + os.RemoveAll(testDir) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + dReceived := deleteReceived.value() + if dReceived < 2 { + t.Fatalf("did not receive at least %d delete events, received %d after 500 ms", 2, dReceived) + } +} + +func TestFsnotifySubDir(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + testFile1 := filepath.Join(testDir, "TestFsnotifyFile1.testfile") + testSubDir := filepath.Join(testDir, "sub") + testSubDirFile := filepath.Join(testDir, "sub/TestFsnotifyFile1.testfile") + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var createReceived, deleteReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testSubDir) || event.Name == filepath.Clean(testFile1) { + t.Logf("event received: %s", event) + if event.Op&Create == Create { + createReceived.increment() + } + if event.Op&Remove == Remove { + deleteReceived.increment() + } + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + addWatch(t, watcher, testDir) + + // Create sub-directory + if err := os.Mkdir(testSubDir, 0777); err != nil { + t.Fatalf("failed to create test sub-directory: %s", err) + } + + // Create a file + var f *os.File + f, err := os.OpenFile(testFile1, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + // Create a file (Should not see this! we are not watching subdir) + var fs *os.File + fs, err = os.OpenFile(testSubDirFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + fs.Sync() + fs.Close() + + time.Sleep(200 * time.Millisecond) + + // Make sure receive deletes for both file and sub-directory + os.RemoveAll(testSubDir) + os.Remove(testFile1) + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + cReceived := createReceived.value() + if cReceived != 2 { + t.Fatalf("incorrect number of create events received after 500 ms (%d vs %d)", cReceived, 2) + } + dReceived := deleteReceived.value() + if dReceived != 2 { + t.Fatalf("incorrect number of delete events received after 500 ms (%d vs %d)", dReceived, 2) + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } +} + +func TestFsnotifyRename(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var renameReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileRenamed) { + if event.Op&Rename == Rename { + renameReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + f.WriteString("data") + f.Sync() + f.Close() + + // Add a watch for testFile + addWatch(t, watcher, testFile) + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if renameReceived.value() == 0 { + t.Fatal("fsnotify rename events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestFsnotifyRenameToCreate(t *testing.T) { + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory to get file + testDirFrom := tempMkdir(t) + defer os.RemoveAll(testDirFrom) + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDirFrom, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var createReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) || event.Name == filepath.Clean(testFileRenamed) { + if event.Op&Create == Create { + createReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if createReceived.value() == 0 { + t.Fatal("fsnotify create events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestFsnotifyRenameToOverwrite(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping test on %q (os.Rename over existing file does not create event).", runtime.GOOS) + } + + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create directory to get file + testDirFrom := tempMkdir(t) + defer os.RemoveAll(testDirFrom) + + testFile := filepath.Join(testDirFrom, "TestFsnotifyEvents.testfile") + testFileRenamed := filepath.Join(testDir, "TestFsnotifyEvents.testfileRenamed") + + // Create a file + var fr *os.File + fr, err := os.OpenFile(testFileRenamed, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + fr.Sync() + fr.Close() + + addWatch(t, watcher, testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + var eventReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testFileRenamed) { + eventReceived.increment() + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + + if err := testRename(testFile, testFileRenamed); err != nil { + t.Fatalf("rename failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + if eventReceived.value() == 0 { + t.Fatal("fsnotify events have not been received after 500 ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(2 * time.Second): + t.Fatal("event stream was not closed after 2 seconds") + } + + os.Remove(testFileRenamed) +} + +func TestRemovalOfWatch(t *testing.T) { + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + watcher := newWatcher(t) + defer watcher.Close() + + addWatch(t, watcher, testDir) + if err := watcher.Remove(testDir); err != nil { + t.Fatalf("Could not remove the watch: %v\n", err) + } + + go func() { + select { + case ev := <-watcher.Events: + t.Fatalf("We received event: %v\n", ev) + case <-time.After(500 * time.Millisecond): + t.Log("No event received, as expected.") + } + }() + + time.Sleep(200 * time.Millisecond) + // Modify the file outside of the watched dir + f, err := os.Open(testFileAlreadyExists) + if err != nil { + t.Fatalf("Open test file failed: %s", err) + } + f.WriteString("data") + f.Sync() + f.Close() + if err := os.Chmod(testFileAlreadyExists, 0700); err != nil { + t.Fatalf("chmod failed: %s", err) + } + time.Sleep(400 * time.Millisecond) +} + +func TestFsnotifyAttrib(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("attributes don't work on Windows.") + } + + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Receive errors on the error channel on a separate goroutine + go func() { + for err := range watcher.Errors { + t.Fatalf("error received: %s", err) + } + }() + + testFile := filepath.Join(testDir, "TestFsnotifyAttrib.testfile") + + // Receive events on the event channel on a separate goroutine + eventstream := watcher.Events + // The modifyReceived counter counts IsModify events that are not IsAttrib, + // and the attribReceived counts IsAttrib events (which are also IsModify as + // a consequence). + var modifyReceived counter + var attribReceived counter + done := make(chan bool) + go func() { + for event := range eventstream { + // Only count relevant events + if event.Name == filepath.Clean(testDir) || event.Name == filepath.Clean(testFile) { + if event.Op&Write == Write { + modifyReceived.increment() + } + if event.Op&Chmod == Chmod { + attribReceived.increment() + } + t.Logf("event received: %s", event) + } else { + t.Logf("unexpected event received: %s", event) + } + } + done <- true + }() + + // Create a file + // This should add at least one event to the fsnotify event queue + var f *os.File + f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + + f.WriteString("data") + f.Sync() + f.Close() + + // Add a watch for testFile + addWatch(t, watcher, testFile) + + if err := os.Chmod(testFile, 0700); err != nil { + t.Fatalf("chmod failed: %s", err) + } + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + // Creating/writing a file changes also the mtime, so IsAttrib should be set to true here + time.Sleep(500 * time.Millisecond) + if modifyReceived.value() != 0 { + t.Fatal("received an unexpected modify event when creating a test file") + } + if attribReceived.value() == 0 { + t.Fatal("fsnotify attribute events have not received after 500 ms") + } + + // Modifying the contents of the file does not set the attrib flag (although eg. the mtime + // might have been modified). + modifyReceived.reset() + attribReceived.reset() + + f, err = os.OpenFile(testFile, os.O_WRONLY, 0) + if err != nil { + t.Fatalf("reopening test file failed: %s", err) + } + + f.WriteString("more data") + f.Sync() + f.Close() + + time.Sleep(500 * time.Millisecond) + + if modifyReceived.value() != 1 { + t.Fatal("didn't receive a modify event after changing test file contents") + } + + if attribReceived.value() != 0 { + t.Fatal("did receive an unexpected attrib event after changing test file contents") + } + + modifyReceived.reset() + attribReceived.reset() + + // Doing a chmod on the file should trigger an event with the "attrib" flag set (the contents + // of the file are not changed though) + if err := os.Chmod(testFile, 0600); err != nil { + t.Fatalf("chmod failed: %s", err) + } + + time.Sleep(500 * time.Millisecond) + + if attribReceived.value() != 1 { + t.Fatal("didn't receive an attribute change after 500ms") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() + t.Log("waiting for the event channel to become closed...") + select { + case <-done: + t.Log("event channel closed") + case <-time.After(1e9): + t.Fatal("event stream was not closed after 1 second") + } + + os.Remove(testFile) +} + +func TestFsnotifyClose(t *testing.T) { + watcher := newWatcher(t) + watcher.Close() + + var done int32 + go func() { + watcher.Close() + atomic.StoreInt32(&done, 1) + }() + + time.Sleep(50e6) // 50 ms + if atomic.LoadInt32(&done) == 0 { + t.Fatal("double Close() test failed: second Close() call didn't return") + } + + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + if err := watcher.Add(testDir); err == nil { + t.Fatal("expected error on Watch() after Close(), got nil") + } +} + +func TestFsnotifyFakeSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks don't work on Windows.") + } + + watcher := newWatcher(t) + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + var errorsReceived counter + // Receive errors on the error channel on a separate goroutine + go func() { + for errors := range watcher.Errors { + t.Logf("Received error: %s", errors) + errorsReceived.increment() + } + }() + + // Count the CREATE events received + var createEventsReceived, otherEventsReceived counter + go func() { + for ev := range watcher.Events { + t.Logf("event received: %s", ev) + if ev.Op&Create == Create { + createEventsReceived.increment() + } else { + otherEventsReceived.increment() + } + } + }() + + addWatch(t, watcher, testDir) + + if err := os.Symlink(filepath.Join(testDir, "zzz"), filepath.Join(testDir, "zzznew")); err != nil { + t.Fatalf("Failed to create bogus symlink: %s", err) + } + t.Logf("Created bogus symlink") + + // We expect this event to be received almost immediately, but let's wait 500 ms to be sure + time.Sleep(500 * time.Millisecond) + + // Should not be error, just no events for broken links (watching nothing) + if errorsReceived.value() > 0 { + t.Fatal("fsnotify errors have been received.") + } + if otherEventsReceived.value() > 0 { + t.Fatal("fsnotify other events received on the broken link") + } + + // Except for 1 create event (for the link itself) + if createEventsReceived.value() == 0 { + t.Fatal("fsnotify create events were not received after 500 ms") + } + if createEventsReceived.value() > 1 { + t.Fatal("fsnotify more create events received than expected") + } + + // Try closing the fsnotify instance + t.Log("calling Close()") + watcher.Close() +} + +// TestConcurrentRemovalOfWatch tests that concurrent calls to RemoveWatch do not race. +// See https://codereview.appspot.com/103300045/ +// go test -test.run=TestConcurrentRemovalOfWatch -test.cpu=1,1,1,1,1 -race +func TestConcurrentRemovalOfWatch(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("regression test for race only present on darwin") + } + + // Create directory to watch + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + // Create a file before watching directory + testFileAlreadyExists := filepath.Join(testDir, "TestFsnotifyEventsExisting.testfile") + { + var f *os.File + f, err := os.OpenFile(testFileAlreadyExists, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + t.Fatalf("creating test file failed: %s", err) + } + f.Sync() + f.Close() + } + + watcher := newWatcher(t) + defer watcher.Close() + + addWatch(t, watcher, testDir) + + // Test that RemoveWatch can be invoked concurrently, with no data races. + removed1 := make(chan struct{}) + go func() { + defer close(removed1) + watcher.Remove(testDir) + }() + removed2 := make(chan struct{}) + go func() { + close(removed2) + watcher.Remove(testDir) + }() + <-removed1 + <-removed2 +} + +func TestClose(t *testing.T) { + // Regression test for #59 bad file descriptor from Close + testDir := tempMkdir(t) + defer os.RemoveAll(testDir) + + watcher := newWatcher(t) + if err := watcher.Add(testDir); err != nil { + t.Fatalf("Expected no error on Add, got %v", err) + } + err := watcher.Close() + if err != nil { + t.Fatalf("Expected no error on Close, got %v.", err) + } +} + +func testRename(file1, file2 string) error { + switch runtime.GOOS { + case "windows", "plan9": + return os.Rename(file1, file2) + default: + cmd := exec.Command("mv", file1, file2) + return cmd.Run() + } +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go new file mode 100644 index 0000000000000..f8a364df9cfcb --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/kqueue.go @@ -0,0 +1,468 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly darwin + +package fsnotify + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + "syscall" + "time" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + done chan bool // Channel for sending a "quit message" to the reader goroutine + + kq int // File descriptor (as returned by the kqueue() syscall). + + mu sync.Mutex // Protects access to watcher data + watches map[string]int // Map of watched file descriptors (key: path). + externalWatches map[string]bool // Map of watches added by user of the library. + dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. + paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. + fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). + isClosed bool // Set to true when Close() is first called +} + +type pathInfo struct { + name string + isDir bool +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + kq, err := kqueue() + if err != nil { + return nil, err + } + + w := &Watcher{ + kq: kq, + watches: make(map[string]int), + dirFlags: make(map[string]uint32), + paths: make(map[int]pathInfo), + fileExists: make(map[string]bool), + externalWatches: make(map[string]bool), + Events: make(chan Event), + Errors: make(chan error), + done: make(chan bool), + } + + go w.readEvents() + return w, nil +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return nil + } + w.isClosed = true + w.mu.Unlock() + + w.mu.Lock() + ws := w.watches + w.mu.Unlock() + + var err error + for name := range ws { + if e := w.Remove(name); e != nil && err == nil { + err = e + } + } + + // Send "quit" message to the reader goroutine: + w.done <- true + + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + w.mu.Lock() + w.externalWatches[name] = true + w.mu.Unlock() + return w.addWatch(name, noteAllEvents) +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + name = filepath.Clean(name) + w.mu.Lock() + watchfd, ok := w.watches[name] + w.mu.Unlock() + if !ok { + return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) + } + + const registerRemove = syscall.EV_DELETE + if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { + return err + } + + syscall.Close(watchfd) + + w.mu.Lock() + isDir := w.paths[watchfd].isDir + delete(w.watches, name) + delete(w.paths, watchfd) + delete(w.dirFlags, name) + w.mu.Unlock() + + // Find all watched paths that are in this directory that are not external. + if isDir { + var pathsToRemove []string + w.mu.Lock() + for _, path := range w.paths { + wdir, _ := filepath.Split(path.name) + if filepath.Clean(wdir) == name { + if !w.externalWatches[path.name] { + pathsToRemove = append(pathsToRemove, path.name) + } + } + } + w.mu.Unlock() + for _, name := range pathsToRemove { + // Since these are internal, not much sense in propagating error + // to the user, as that will just confuse them with an error about + // a path they did not explicitly watch themselves. + w.Remove(name) + } + } + + return nil +} + +// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) +const noteAllEvents = syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_ATTRIB | syscall.NOTE_RENAME + +// keventWaitTime to block on each read from kevent +var keventWaitTime = durationToTimespec(100 * time.Millisecond) + +// addWatch adds name to the watched file set. +// The flags are interpreted as described in kevent(2). +func (w *Watcher) addWatch(name string, flags uint32) error { + var isDir bool + // Make ./name and name equivalent + name = filepath.Clean(name) + + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return errors.New("kevent instance already closed") + } + watchfd, alreadyWatching := w.watches[name] + // We already have a watch, but we can still override flags. + if alreadyWatching { + isDir = w.paths[watchfd].isDir + } + w.mu.Unlock() + + if !alreadyWatching { + fi, err := os.Lstat(name) + if err != nil { + return err + } + + // Don't watch sockets. + if fi.Mode()&os.ModeSocket == os.ModeSocket { + return nil + } + + // Don't watch named pipes. + if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { + return nil + } + + // Follow Symlinks + // Unfortunately, Linux can add bogus symlinks to watch list without + // issue, and Windows can't do symlinks period (AFAIK). To maintain + // consistency, we will act like everything is fine. There will simply + // be no file events for broken symlinks. + // Hence the returns of nil on errors. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + name, err = filepath.EvalSymlinks(name) + if err != nil { + return nil + } + + fi, err = os.Lstat(name) + if err != nil { + return nil + } + } + + watchfd, err = syscall.Open(name, openMode, 0700) + if watchfd == -1 { + return err + } + + isDir = fi.IsDir() + } + + const registerAdd = syscall.EV_ADD | syscall.EV_CLEAR | syscall.EV_ENABLE + if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { + syscall.Close(watchfd) + return err + } + + if !alreadyWatching { + w.mu.Lock() + w.watches[name] = watchfd + w.paths[watchfd] = pathInfo{name: name, isDir: isDir} + w.mu.Unlock() + } + + if isDir { + // Watch the directory if it has not been watched before, + // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) + w.mu.Lock() + watchDir := (flags&syscall.NOTE_WRITE) == syscall.NOTE_WRITE && + (!alreadyWatching || (w.dirFlags[name]&syscall.NOTE_WRITE) != syscall.NOTE_WRITE) + // Store flags so this watch can be updated later + w.dirFlags[name] = flags + w.mu.Unlock() + + if watchDir { + if err := w.watchDirectoryFiles(name); err != nil { + return err + } + } + } + return nil +} + +// readEvents reads from kqueue and converts the received kevents into +// Event values that it sends down the Events channel. +func (w *Watcher) readEvents() { + eventBuffer := make([]syscall.Kevent_t, 10) + + for { + // See if there is a message on the "done" channel + select { + case <-w.done: + err := syscall.Close(w.kq) + if err != nil { + w.Errors <- err + } + close(w.Events) + close(w.Errors) + return + default: + } + + // Get new events + kevents, err := read(w.kq, eventBuffer, &keventWaitTime) + // EINTR is okay, the syscall was interrupted before timeout expired. + if err != nil && err != syscall.EINTR { + w.Errors <- err + continue + } + + // Flush the events we received to the Events channel + for len(kevents) > 0 { + kevent := &kevents[0] + watchfd := int(kevent.Ident) + mask := uint32(kevent.Fflags) + w.mu.Lock() + path := w.paths[watchfd] + w.mu.Unlock() + event := newEvent(path.name, mask) + + if path.isDir && !(event.Op&Remove == Remove) { + // Double check to make sure the directory exists. This can happen when + // we do a rm -fr on a recursively watched folders and we receive a + // modification event first but the folder has been deleted and later + // receive the delete event + if _, err := os.Lstat(event.Name); os.IsNotExist(err) { + // mark is as delete event + event.Op |= Remove + } + } + + if event.Op&Rename == Rename || event.Op&Remove == Remove { + w.Remove(event.Name) + w.mu.Lock() + delete(w.fileExists, event.Name) + w.mu.Unlock() + } + + if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { + w.sendDirectoryChangeEvents(event.Name) + } else { + // Send the event on the Events channel + w.Events <- event + } + + if event.Op&Remove == Remove { + // Look for a file that may have overwritten this. + // For example, mv f1 f2 will delete f2, then create f2. + fileDir, _ := filepath.Split(event.Name) + fileDir = filepath.Clean(fileDir) + w.mu.Lock() + _, found := w.watches[fileDir] + w.mu.Unlock() + if found { + // make sure the directory exists before we watch for changes. When we + // do a recursive watch and perform rm -fr, the parent directory might + // have gone missing, ignore the missing directory and let the + // upcoming delete event remove the watch from the parent directory. + if _, err := os.Lstat(fileDir); os.IsExist(err) { + w.sendDirectoryChangeEvents(fileDir) + // FIXME: should this be for events on files or just isDir? + } + } + } + + // Move to next event + kevents = kevents[1:] + } + } +} + +// newEvent returns an platform-independent Event based on kqueue Fflags. +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&syscall.NOTE_DELETE == syscall.NOTE_DELETE { + e.Op |= Remove + } + if mask&syscall.NOTE_WRITE == syscall.NOTE_WRITE { + e.Op |= Write + } + if mask&syscall.NOTE_RENAME == syscall.NOTE_RENAME { + e.Op |= Rename + } + if mask&syscall.NOTE_ATTRIB == syscall.NOTE_ATTRIB { + e.Op |= Chmod + } + return e +} + +func newCreateEvent(name string) Event { + return Event{Name: name, Op: Create} +} + +// watchDirectoryFiles to mimic inotify when adding a watch on a directory +func (w *Watcher) watchDirectoryFiles(dirPath string) error { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + return err + } + + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + if err := w.internalWatch(filePath, fileInfo); err != nil { + return err + } + + w.mu.Lock() + w.fileExists[filePath] = true + w.mu.Unlock() + } + + return nil +} + +// sendDirectoryEvents searches the directory for newly created files +// and sends them over the event channel. This functionality is to have +// the BSD version of fsnotify match Linux inotify which provides a +// create event for files created in a watched directory. +func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + w.Errors <- err + } + + // Search for new files + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + w.mu.Lock() + _, doesExist := w.fileExists[filePath] + w.mu.Unlock() + if !doesExist { + // Send create event + w.Events <- newCreateEvent(filePath) + } + + // like watchDirectoryFiles (but without doing another ReadDir) + if err := w.internalWatch(filePath, fileInfo); err != nil { + return + } + + w.mu.Lock() + w.fileExists[filePath] = true + w.mu.Unlock() + } +} + +func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) error { + if fileInfo.IsDir() { + // mimic Linux providing delete events for subdirectories + // but preserve the flags used if currently watching subdirectory + w.mu.Lock() + flags := w.dirFlags[name] + w.mu.Unlock() + + flags |= syscall.NOTE_DELETE + return w.addWatch(name, flags) + } + + // watch file to mimic Linux inotify + return w.addWatch(name, noteAllEvents) +} + +// kqueue creates a new kernel event queue and returns a descriptor. +func kqueue() (kq int, err error) { + kq, err = syscall.Kqueue() + if kq == -1 { + return kq, err + } + return kq, nil +} + +// register events with the queue +func register(kq int, fds []int, flags int, fflags uint32) error { + changes := make([]syscall.Kevent_t, len(fds)) + + for i, fd := range fds { + // SetKevent converts int to the platform-specific types: + syscall.SetKevent(&changes[i], fd, syscall.EVFILT_VNODE, flags) + changes[i].Fflags = fflags + } + + // register the events + success, err := syscall.Kevent(kq, changes, nil, nil) + if success == -1 { + return err + } + return nil +} + +// read retrieves pending events, or waits until an event occurs. +// A timeout of nil blocks indefinitely, while 0 polls the queue. +func read(kq int, events []syscall.Kevent_t, timeout *syscall.Timespec) ([]syscall.Kevent_t, error) { + n, err := syscall.Kevent(kq, nil, events, timeout) + if err != nil { + return nil, err + } + return events[0:n], nil +} + +// durationToTimespec prepares a timeout value +func durationToTimespec(d time.Duration) syscall.Timespec { + return syscall.NsecToTimespec(d.Nanoseconds()) +} diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_bsd.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_bsd.go new file mode 100644 index 0000000000000..c57ccb427b911 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_bsd.go @@ -0,0 +1,11 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly + +package fsnotify + +import "syscall" + +const openMode = syscall.O_NONBLOCK | syscall.O_RDONLY diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_darwin.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_darwin.go new file mode 100644 index 0000000000000..174b2c331f1a1 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/open_mode_darwin.go @@ -0,0 +1,12 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package fsnotify + +import "syscall" + +// note: this constant is not defined on BSD +const openMode = syscall.O_EVTONLY diff --git a/Godeps/_workspace/src/gopkg.in/fsnotify.v1/windows.go b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/windows.go new file mode 100644 index 0000000000000..811585227d89b --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/fsnotify.v1/windows.go @@ -0,0 +1,561 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "syscall" + "unsafe" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + isClosed bool // Set to true when Close() is first called + mu sync.Mutex // Map access + port syscall.Handle // Handle to completion port + watches watchMap // Map of watches (key: i-number) + input chan *input // Inputs to the reader are sent on this channel + quit chan chan<- error +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if e != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", e) + } + w := &Watcher{ + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + Events: make(chan Event, 50), + Errors: make(chan error), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + return w, nil +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + if w.isClosed { + return errors.New("watcher already closed") + } + in := &input{ + op: opAddWatch, + path: filepath.Clean(name), + flags: sys_FS_ALL_EVENTS, + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(name), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +const ( + // Options for AddWatch + sys_FS_ONESHOT = 0x80000000 + sys_FS_ONLYDIR = 0x1000000 + + // Events + sys_FS_ACCESS = 0x1 + sys_FS_ALL_EVENTS = 0xfff + sys_FS_ATTRIB = 0x4 + sys_FS_CLOSE = 0x18 + sys_FS_CREATE = 0x100 + sys_FS_DELETE = 0x200 + sys_FS_DELETE_SELF = 0x400 + sys_FS_MODIFY = 0x2 + sys_FS_MOVE = 0xc0 + sys_FS_MOVED_FROM = 0x40 + sys_FS_MOVED_TO = 0x80 + sys_FS_MOVE_SELF = 0x800 + + // Special events + sys_FS_IGNORED = 0x8000 + sys_FS_Q_OVERFLOW = 0x4000 +) + +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&sys_FS_CREATE == sys_FS_CREATE || mask&sys_FS_MOVED_TO == sys_FS_MOVED_TO { + e.Op |= Create + } + if mask&sys_FS_DELETE == sys_FS_DELETE || mask&sys_FS_DELETE_SELF == sys_FS_DELETE_SELF { + e.Op |= Remove + } + if mask&sys_FS_MODIFY == sys_FS_MODIFY { + e.Op |= Write + } + if mask&sys_FS_MOVE == sys_FS_MOVE || mask&sys_FS_MOVE_SELF == sys_FS_MOVE_SELF || mask&sys_FS_MOVED_FROM == sys_FS_MOVED_FROM { + e.Op |= Rename + } + if mask&sys_FS_ATTRIB == sys_FS_ATTRIB { + e.Op |= Chmod + } + return e +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + reply chan error +} + +type inode struct { + handle syscall.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov syscall.Overlapped + ino *inode // i-number + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf [4096]byte +} + +type indexMap map[uint64]*watch +type watchMap map[uint32]indexMap + +func (w *Watcher) wakeupReader() error { + e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if e != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", e) + } + return nil +} + +func getDir(pathname string) (dir string, err error) { + attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) + if e != nil { + return "", os.NewSyscallError("GetFileAttributes", e) + } + if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func getIno(path string) (ino *inode, err error) { + h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), + syscall.FILE_LIST_DIRECTORY, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) + if e != nil { + return nil, os.NewSyscallError("CreateFile", e) + } + var fi syscall.ByHandleFileInformation + if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { + syscall.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", e) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *Watcher) addWatch(pathname string, flags uint64) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + if flags&sys_FS_ONLYDIR != 0 && pathname != dir { + return nil + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { + syscall.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", e) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + syscall.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + if err = w.startRead(watchEntry); err != nil { + return err + } + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *Watcher) remWatch(pathname string) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + if watch == nil { + return fmt.Errorf("can't remove non-existent watch for: %s", pathname) + } + if pathname == dir { + w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(watch.path+"\\"+name, watch.names[name]&sys_FS_IGNORED) + delete(watch.names, name) + } + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *Watcher) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(watch.path+"\\"+name, mask&sys_FS_IGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *Watcher) startRead(watch *watch) error { + if e := syscall.CancelIo(watch.ino.handle); e != nil { + w.Errors <- os.NewSyscallError("CancelIo", e) + w.deleteWatch(watch) + } + mask := toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= toWindowsFlags(m) + } + if mask == 0 { + if e := syscall.CloseHandle(watch.ino.handle); e != nil { + w.Errors <- os.NewSyscallError("CloseHandle", e) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], + uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) + if e != nil { + err := os.NewSyscallError("ReadDirectoryChanges", e) + if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + if w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) { + if watch.mask&sys_FS_ONESHOT != 0 { + watch.mask = 0 + } + } + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Events channel. +// Entry point to the I/O thread. +func (w *Watcher) readEvents() { + var ( + n, key uint32 + ov *syscall.Overlapped + ) + runtime.LockOSThread() + + for { + e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) + watch := (*watch)(unsafe.Pointer(ov)) + + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + var err error + if e := syscall.CloseHandle(w.port); e != nil { + err = os.NewSyscallError("CloseHandle", e) + } + close(w.Events) + close(w.Errors) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags)) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch e { + case syscall.ERROR_MORE_DATA: + if watch == nil { + w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case syscall.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case syscall.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) + continue + case nil: + } + + var offset uint32 + for { + if n == 0 { + w.Events <- newEvent("", sys_FS_Q_OVERFLOW) + w.Errors <- errors.New("short read in readEvents()") + break + } + + // Point "raw" to the event in the buffer + raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) + name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) + fullname := watch.path + "\\" + name + + var mask uint64 + switch raw.Action { + case syscall.FILE_ACTION_REMOVED: + mask = sys_FS_DELETE_SELF + case syscall.FILE_ACTION_MODIFIED: + mask = sys_FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sys_FS_MOVE_SELF + } + } + + sendNameEvent := func() { + if w.sendEvent(fullname, watch.names[name]&mask) { + if watch.names[name]&sys_FS_ONESHOT != 0 { + delete(watch.names, name) + } + } + } + if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { + sendNameEvent() + } + if raw.Action == syscall.FILE_ACTION_REMOVED { + w.sendEvent(fullname, watch.names[name]&sys_FS_IGNORED) + delete(watch.names, name) + } + if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { + if watch.mask&sys_FS_ONESHOT != 0 { + watch.mask = 0 + } + } + if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { + fullname = watch.path + "\\" + watch.rename + sendNameEvent() + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") + break + } + } + + if err := w.startRead(watch); err != nil { + w.Errors <- err + } + } +} + +func (w *Watcher) sendEvent(name string, mask uint64) bool { + if mask == 0 { + return false + } + event := newEvent(name, uint32(mask)) + select { + case ch := <-w.quit: + w.quit <- ch + case w.Events <- event: + } + return true +} + +func toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sys_FS_ACCESS != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS + } + if mask&sys_FS_MODIFY != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&sys_FS_ATTRIB != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES + } + if mask&(sys_FS_MOVE|sys_FS_CREATE|sys_FS_DELETE) != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func toFSnotifyFlags(action uint32) uint64 { + switch action { + case syscall.FILE_ACTION_ADDED: + return sys_FS_CREATE + case syscall.FILE_ACTION_REMOVED: + return sys_FS_DELETE + case syscall.FILE_ACTION_MODIFIED: + return sys_FS_MODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + return sys_FS_MOVED_FROM + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + return sys_FS_MOVED_TO + } + return 0 +} diff --git a/Godeps/_workspace/src/gopkg.in/tomb.v1/LICENSE b/Godeps/_workspace/src/gopkg.in/tomb.v1/LICENSE new file mode 100644 index 0000000000000..a4249bb31dd45 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/tomb.v1/LICENSE @@ -0,0 +1,29 @@ +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/gopkg.in/tomb.v1/README.md b/Godeps/_workspace/src/gopkg.in/tomb.v1/README.md new file mode 100644 index 0000000000000..3ae8788e81311 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/tomb.v1/README.md @@ -0,0 +1,4 @@ +Installation and usage +---------------------- + +See [gopkg.in/tomb.v1](https://gopkg.in/tomb.v1) for documentation and usage details. diff --git a/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb.go b/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb.go new file mode 100644 index 0000000000000..9aec56d821db6 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb.go @@ -0,0 +1,176 @@ +// Copyright (c) 2011 - Gustavo Niemeyer +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The tomb package offers a conventional API for clean goroutine termination. +// +// A Tomb tracks the lifecycle of a goroutine as alive, dying or dead, +// and the reason for its death. +// +// The zero value of a Tomb assumes that a goroutine is about to be +// created or already alive. Once Kill or Killf is called with an +// argument that informs the reason for death, the goroutine is in +// a dying state and is expected to terminate soon. Right before the +// goroutine function or method returns, Done must be called to inform +// that the goroutine is indeed dead and about to stop running. +// +// A Tomb exposes Dying and Dead channels. These channels are closed +// when the Tomb state changes in the respective way. They enable +// explicit blocking until the state changes, and also to selectively +// unblock select statements accordingly. +// +// When the tomb state changes to dying and there's still logic going +// on within the goroutine, nested functions and methods may choose to +// return ErrDying as their error value, as this error won't alter the +// tomb state if provided to the Kill method. This is a convenient way to +// follow standard Go practices in the context of a dying tomb. +// +// For background and a detailed example, see the following blog post: +// +// http://blog.labix.org/2011/10/09/death-of-goroutines-under-control +// +// For a more complex code snippet demonstrating the use of multiple +// goroutines with a single Tomb, see: +// +// http://play.golang.org/p/Xh7qWsDPZP +// +package tomb + +import ( + "errors" + "fmt" + "sync" +) + +// A Tomb tracks the lifecycle of a goroutine as alive, dying or dead, +// and the reason for its death. +// +// See the package documentation for details. +type Tomb struct { + m sync.Mutex + dying chan struct{} + dead chan struct{} + reason error +} + +var ( + ErrStillAlive = errors.New("tomb: still alive") + ErrDying = errors.New("tomb: dying") +) + +func (t *Tomb) init() { + t.m.Lock() + if t.dead == nil { + t.dead = make(chan struct{}) + t.dying = make(chan struct{}) + t.reason = ErrStillAlive + } + t.m.Unlock() +} + +// Dead returns the channel that can be used to wait +// until t.Done has been called. +func (t *Tomb) Dead() <-chan struct{} { + t.init() + return t.dead +} + +// Dying returns the channel that can be used to wait +// until t.Kill or t.Done has been called. +func (t *Tomb) Dying() <-chan struct{} { + t.init() + return t.dying +} + +// Wait blocks until the goroutine is in a dead state and returns the +// reason for its death. +func (t *Tomb) Wait() error { + t.init() + <-t.dead + t.m.Lock() + reason := t.reason + t.m.Unlock() + return reason +} + +// Done flags the goroutine as dead, and should be called a single time +// right before the goroutine function or method returns. +// If the goroutine was not already in a dying state before Done is +// called, it will be flagged as dying and dead at once with no +// error. +func (t *Tomb) Done() { + t.Kill(nil) + close(t.dead) +} + +// Kill flags the goroutine as dying for the given reason. +// Kill may be called multiple times, but only the first +// non-nil error is recorded as the reason for termination. +// +// If reason is ErrDying, the previous reason isn't replaced +// even if it is nil. It's a runtime error to call Kill with +// ErrDying if t is not in a dying state. +func (t *Tomb) Kill(reason error) { + t.init() + t.m.Lock() + defer t.m.Unlock() + if reason == ErrDying { + if t.reason == ErrStillAlive { + panic("tomb: Kill with ErrDying while still alive") + } + return + } + if t.reason == nil || t.reason == ErrStillAlive { + t.reason = reason + } + // If the receive on t.dying succeeds, then + // it can only be because we have already closed it. + // If it blocks, then we know that it needs to be closed. + select { + case <-t.dying: + default: + close(t.dying) + } +} + +// Killf works like Kill, but builds the reason providing the received +// arguments to fmt.Errorf. The generated error is also returned. +func (t *Tomb) Killf(f string, a ...interface{}) error { + err := fmt.Errorf(f, a...) + t.Kill(err) + return err +} + +// Err returns the reason for the goroutine death provided via Kill +// or Killf, or ErrStillAlive when the goroutine is still alive. +func (t *Tomb) Err() (reason error) { + t.init() + t.m.Lock() + reason = t.reason + t.m.Unlock() + return +} diff --git a/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb_test.go b/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb_test.go new file mode 100644 index 0000000000000..71f11fafba848 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/tomb.v1/tomb_test.go @@ -0,0 +1,114 @@ +package tomb_test + +import ( + "errors" + "gopkg.in/tomb.v1" + "reflect" + "testing" +) + +func TestNewTomb(t *testing.T) { + tb := &tomb.Tomb{} + testState(t, tb, false, false, tomb.ErrStillAlive) + + tb.Done() + testState(t, tb, true, true, nil) +} + +func TestKill(t *testing.T) { + // a nil reason flags the goroutine as dying + tb := &tomb.Tomb{} + tb.Kill(nil) + testState(t, tb, true, false, nil) + + // a non-nil reason now will override Kill + err := errors.New("some error") + tb.Kill(err) + testState(t, tb, true, false, err) + + // another non-nil reason won't replace the first one + tb.Kill(errors.New("ignore me")) + testState(t, tb, true, false, err) + + tb.Done() + testState(t, tb, true, true, err) +} + +func TestKillf(t *testing.T) { + tb := &tomb.Tomb{} + + err := tb.Killf("BO%s", "OM") + if s := err.Error(); s != "BOOM" { + t.Fatalf(`Killf("BO%s", "OM"): want "BOOM", got %q`, s) + } + testState(t, tb, true, false, err) + + // another non-nil reason won't replace the first one + tb.Killf("ignore me") + testState(t, tb, true, false, err) + + tb.Done() + testState(t, tb, true, true, err) +} + +func TestErrDying(t *testing.T) { + // ErrDying being used properly, after a clean death. + tb := &tomb.Tomb{} + tb.Kill(nil) + tb.Kill(tomb.ErrDying) + testState(t, tb, true, false, nil) + + // ErrDying being used properly, after an errorful death. + err := errors.New("some error") + tb.Kill(err) + tb.Kill(tomb.ErrDying) + testState(t, tb, true, false, err) + + // ErrDying being used badly, with an alive tomb. + tb = &tomb.Tomb{} + defer func() { + err := recover() + if err != "tomb: Kill with ErrDying while still alive" { + t.Fatalf("Wrong panic on Kill(ErrDying): %v", err) + } + testState(t, tb, false, false, tomb.ErrStillAlive) + }() + tb.Kill(tomb.ErrDying) +} + +func testState(t *testing.T, tb *tomb.Tomb, wantDying, wantDead bool, wantErr error) { + select { + case <-tb.Dying(): + if !wantDying { + t.Error("<-Dying: should block") + } + default: + if wantDying { + t.Error("<-Dying: should not block") + } + } + seemsDead := false + select { + case <-tb.Dead(): + if !wantDead { + t.Error("<-Dead: should block") + } + seemsDead = true + default: + if wantDead { + t.Error("<-Dead: should not block") + } + } + if err := tb.Err(); err != wantErr { + t.Errorf("Err: want %#v, got %#v", wantErr, err) + } + if wantDead && seemsDead { + waitErr := tb.Wait() + switch { + case waitErr == tomb.ErrStillAlive: + t.Errorf("Wait should not return ErrStillAlive") + case !reflect.DeepEqual(waitErr, wantErr): + t.Errorf("Wait: want %#v, got %#v", wantErr, waitErr) + } + } +} diff --git a/plugins/all/all.go b/plugins/all/all.go index f29a4987b3b6c..648b8446c238c 100644 --- a/plugins/all/all.go +++ b/plugins/all/all.go @@ -11,6 +11,7 @@ import ( _ "github.com/influxdb/telegraf/plugins/httpjson" _ "github.com/influxdb/telegraf/plugins/kafka_consumer" _ "github.com/influxdb/telegraf/plugins/leofs" + _ "github.com/influxdb/telegraf/plugins/logstreamer" _ "github.com/influxdb/telegraf/plugins/lustre2" _ "github.com/influxdb/telegraf/plugins/memcached" _ "github.com/influxdb/telegraf/plugins/mongodb" diff --git a/plugins/logstreamer/logstream/files.go b/plugins/logstreamer/logstream/files.go new file mode 100644 index 0000000000000..9fd5ca06f25ea --- /dev/null +++ b/plugins/logstreamer/logstream/files.go @@ -0,0 +1,203 @@ +package logstream + +import ( + "fmt" + "os" + "path/filepath" + + log "github.com/Sirupsen/logrus" + + "github.com/howeyc/fsnotify" + "github.com/hpcloud/tail" +) + +func walkLogDir(dir string) (files []string, err error) { + log.Debugf("Walking %s", dir) + if string(dir[len(dir)-1]) != "/" { + dir = dir + "/" + } + visit := func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + return nil + } + abs, err := filepath.Abs(path) + if err != nil { + log.Fatalf("Can't get abs filepath: %s", err) + } + files = append(files, abs) + return nil + } + err = filepath.Walk(dir, visit) + return +} + +func WatchFiles(dirs []string, groups []Group, continuous, readOnce bool) chan Metric { + + // get list of all files in watch dir + files := make([]string, 0) + for _, dir := range dirs { + fs, err := walkLogDir(dir) + if err != nil { + panic(err) + } + for _, f := range fs { + files = append(files, f) + } + } + + // assign file per group + assignedFiles, err := assignFiles(files, groups) + if err != nil { + log.Fatalln("can't assign file per group", err) + } + + doneCh := make(chan string) + assignedFilesCount := len(assignedFiles) + log.Debugf("Assigned files count: %d", assignedFilesCount) + + for _, file := range assignedFiles { + file.doneCh = doneCh + go file.tail() + } + + if continuous { + for _, dir := range dirs { + go continueWatch(&dir, groups) + } + } + + go func() { + for name := range doneCh { + log.Debugf("Finished reading %s", name) + } + }() + + return metricsChan + +} + +func assignFiles(files []string, groups []Group) (outFiles []*File, err error) { + for n, group := range groups { + log.Debugf("Assigning files for group: %s", group.Mask) + var assignedFiles []*File + if group.Name == "" { + group.Name = fmt.Sprintf("group%d", n) + + } + if assignedFiles, err = getFilesByGroup(files, &group); err == nil { + for _, assignedFile := range assignedFiles { + outFiles = append(outFiles, assignedFile) + } + } else { + return + } + } + return +} + +func getFilesByGroup(allFiles []string, group *Group) ([]*File, error) { + files := make([]*File, 0) + regex := *group.Mask + for _, f := range allFiles { + if !regex.MatchString(filepath.Base(f)) { + continue + } + file, err := NewFile(f, group) + if err != nil { + return files, err + } + files = append(files, file) + } + return files, nil +} + +func continueWatch(dir *string, groups []Group) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("Can't create watcher: %s", err) + } + + done := make(chan bool) + + // Process events + go func() { + for { + select { + case ev := <-watcher.Event: + if ev.IsCreate() { + files := make([]string, 0) + file, err := filepath.Abs(ev.Name) + if err != nil { + log.Printf("can't get file %+v", err) + continue + } + files = append(files, file) + log.Debugf("Assigning %s", file) + assignFiles(files, groups) + } + case err := <-watcher.Error: + log.Println("error:", err) + } + } + }() + + err = watcher.Watch(*dir) + if err != nil { + log.Fatalf("Can't watch directory: %s", err) + } + + <-done + + /* ... do stuff ... */ + watcher.Close() +} + +func NewFile(fpath string, group *Group) (*File, error) { + file := &File{group: group} + var ( + err error + whence int + ) + logger := log.StandardLogger() + + // if we have DateFormat, then we can read file as many times as we want + // without duplicating points + if group.DateFormat != "" { + log.Debugf("Read whole file from the beginning: %+v", fpath) + whence = 0 + } else { + log.Debugf("Read file from the end: %+v", fpath) + whence = 2 + } + seekInfo := &tail.SeekInfo{Offset: 0, Whence: whence} + + file.Tail, err = tail.TailFile(fpath, tail.Config{Follow: true, ReOpen: true, Location: seekInfo, Logger: logger}) + return file, err +} + +type File struct { + Tail *tail.Tail + group *Group + doneCh chan string +} + +func (f *File) tail() { + log.Printf("Start tailing %+v", f.Tail.Filename) + defer func() { f.doneCh <- f.Tail.Filename }() + for line := range f.Tail.Lines { + f.checkLineRules(&line.Text) + } +} + +func (f *File) checkLineRule(line *string, rule *Rule) { + match := rule.Match(line) + if match != nil { + rule.process(f.group, match) + } +} + +func (f *File) checkLineRules(line *string) { + for _, rule := range f.group.Rules { + f.checkLineRule(line, rule) + } +} diff --git a/plugins/logstreamer/logstream/metric.go b/plugins/logstreamer/logstream/metric.go new file mode 100644 index 0000000000000..0c1985af33a2c --- /dev/null +++ b/plugins/logstreamer/logstream/metric.go @@ -0,0 +1,15 @@ +package logstream + +import "time" + +var metricsChan = make(chan Metric, 100) + +func GetMetrics() chan Metric { + return metricsChan +} + +type Metric struct { + Group *Group + Match map[string]string + Date time.Time +} diff --git a/plugins/logstreamer/logstream/rule.go b/plugins/logstreamer/logstream/rule.go new file mode 100644 index 0000000000000..1ef59eae101d1 --- /dev/null +++ b/plugins/logstreamer/logstream/rule.go @@ -0,0 +1,82 @@ +package logstream + +import ( + "regexp" + "strings" + "time" + + log "github.com/Sirupsen/logrus" +) + +type Group struct { + Mask *Mask + Rules []*Rule + Name string + DateFormat string +} + +type Mask struct { + *regexp.Regexp +} + +func (m *Mask) UnmarshalTOML(data []byte) (err error) { + m.Regexp, err = regexp.Compile(strings.Trim(string(data), "\"'")) + return err +} + +func NewRule(sregexp string) (*Rule, error) { + rule := &Rule{} + var err error + if rule.regexp, err = regexp.Compile(sregexp); err != nil { + return rule, err + } + rule.subexpNames = rule.regexp.SubexpNames() + return rule, nil +} + +type Rule struct { + regexp *regexp.Regexp + subexpNames []string +} + +func (rule *Rule) UnmarshalTOML(data []byte) (err error) { + rule.regexp, err = regexp.Compile(strings.Trim(string(data), "\"'")) + if err == nil { + rule.subexpNames = rule.regexp.SubexpNames() + } + return err +} + +func (rule *Rule) Match(line *string) map[string]string { + matches := rule.regexp.FindStringSubmatch(*line) + if len(matches) == 0 { + return nil + } + + // TODO: cache subexnames + out := make(map[string]string) + for i, value := range matches[1:] { + out[rule.subexpNames[i+1]] = value + } + if len(out) > 0 { + return out + } + return nil +} + +func (rule *Rule) process(group *Group, match map[string]string) { + metric := Metric{Group: group} + if group.DateFormat != "" { + if rawDate, ok := match["date"]; ok { + if date, err := time.Parse(group.DateFormat, rawDate); err == nil { + delete(match, "date") + metric.Date = date + } else { + log.Errorf("Can't parse date '%s': %s", rawDate, err) + return + } + } + } + metric.Match = match + metricsChan <- metric +} diff --git a/plugins/logstreamer/logstreamer.go b/plugins/logstreamer/logstreamer.go new file mode 100644 index 0000000000000..fddd565d31f7f --- /dev/null +++ b/plugins/logstreamer/logstreamer.go @@ -0,0 +1,129 @@ +/* +Logstream telegraf plugin + +*/ +package logstreamer + +import ( + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/influxdb/telegraf/plugins" + "github.com/influxdb/telegraf/plugins/logstreamer/logstream" +) + +// Lustre proc files can change between versions, so we want to future-proof +// by letting people choose what to look at. +type Logstreamer struct { + filename string `toml:-` + Groups []logstream.Group `toml:"group"` + Dirs []string + Log string + Debug bool + metrics chan logstream.Metric + init sync.Once +} + +var sampleConfig = ` + #dirs = ["/var/log/nginx/"] + #debug = false + + [[logstreamer.group]] + mask = "^.*log$" + rules = ['\s\[(?P\d{1,2}/\w*/\d+:\d+:\d+:\d+ [+-]?\d+)\]\s.*?"\s(?P\d{3})\s(?P\d+)'] + name = "nginx" + + # Write the date below in your format + # Mon Jan 2 15:04:05 -0700 MST 2006 + date_format = "02/Jan/2006:15:04:05 -0700" +` + +// SampleConfig returns sample configuration message +func (l *Logstreamer) SampleConfig() string { + return sampleConfig +} + +// Description returns description of Lustre2 plugin +func (l *Logstreamer) Description() string { + return "Gets metrics from local logs, parsing them using regular expressions" +} + +// Gather reads stats from all lustre targets +func (l *Logstreamer) Gather(acc plugins.Accumulator) error { + l.init.Do(func() { + l.metrics = logstream.WatchFiles(l.Dirs, l.Groups, false, false) + }) + timeout := time.After(1 * time.Second) + errs := make([]string, 0) + for { + select { + case <-timeout: + goto out + case metric := <-l.metrics: + if err := l.addMetric(metric, acc); err != nil { + errs = append(errs, err.Error()) + } + } + } +out: + + if len(errs) != 0 { + return fmt.Errorf("Can't add metric: %s", strings.Join(errs, ", ")) + } + + return nil +} + +func (l *Logstreamer) addMetric(metric logstream.Metric, acc plugins.Accumulator) (err error) { + var date time.Time + if !metric.Date.IsZero() { // compare with zero time + date = metric.Date + } else { + date = time.Now() + } + tags := make(map[string]string) + tags["group"] = metric.Group.Name + + values := make(map[string]interface{}) + + for key, value := range metric.Match { + key = strings.ToLower(key) + if strings.HasSuffix(key, "_value") { + // try to parse numerical value + i, err := strconv.ParseInt(value, 10, 64) + if err == nil { + values[strings.TrimSuffix(key, "_value")] = i + continue + } + f, err := strconv.ParseFloat(value, 64) + if err == nil { + values[strings.TrimSuffix(key, "_value")] = f + continue + } + + return fmt.Errorf("Can't parse '%s:%s' as numeric value", key, value, err) + + } else { + tags[key] = value + } + } + + acc.AddFields( + metric.Group.Name, + values, + tags, + date, + ) + + return + +} + +func init() { + plugins.Add("logstreamer", func() plugins.Plugin { + return &Logstreamer{} + }) +}