Skip to content

Commit

Permalink
[mac] Regression fixes for native fsevents (#717)
Browse files Browse the repository at this point in the history
  • Loading branch information
@CCP-Aporia
CCP-Aporia authored Dec 10, 2020
1 parent 7f7937d commit 614ccfc
Show file tree
Hide file tree
Showing 2 changed files with 132 additions and 14 deletions.
99 changes: 87 additions & 12 deletions src/watchdog_fsevents.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,29 @@ PyObject* NativeEventTypeID(PyObject* instance, void* closure)
Py_RETURN_FALSE; \
}

FLAG_PROPERTY(IsMustScanSubDirs, kFSEventStreamEventFlagMustScanSubDirs)
FLAG_PROPERTY(IsUserDropped, kFSEventStreamEventFlagUserDropped)
FLAG_PROPERTY(IsKernelDropped, kFSEventStreamEventFlagKernelDropped)
FLAG_PROPERTY(IsEventIdsWrapped, kFSEventStreamEventFlagEventIdsWrapped)
FLAG_PROPERTY(IsHistoryDone, kFSEventStreamEventFlagHistoryDone)
FLAG_PROPERTY(IsRootChanged, kFSEventStreamEventFlagRootChanged)
FLAG_PROPERTY(IsMount, kFSEventStreamEventFlagMount)
FLAG_PROPERTY(IsUnmount, kFSEventStreamEventFlagUnmount)
FLAG_PROPERTY(IsCreated, kFSEventStreamEventFlagItemCreated)
FLAG_PROPERTY(IsRemoved, kFSEventStreamEventFlagItemRemoved)
FLAG_PROPERTY(IsInodeMetaMod, kFSEventStreamEventFlagItemInodeMetaMod)
FLAG_PROPERTY(IsRenamed, kFSEventStreamEventFlagItemRenamed)
FLAG_PROPERTY(IsModified, kFSEventStreamEventFlagItemModified)
FLAG_PROPERTY(IsItemFinderInfoMod, kFSEventStreamEventFlagItemFinderInfoMod)
FLAG_PROPERTY(IsChangeOwner, kFSEventStreamEventFlagItemChangeOwner)
FLAG_PROPERTY(IsXattrMod, kFSEventStreamEventFlagItemXattrMod)
FLAG_PROPERTY(IsFile, kFSEventStreamEventFlagItemIsFile)
FLAG_PROPERTY(IsDirectory, kFSEventStreamEventFlagItemIsDir)
FLAG_PROPERTY(IsSymlink, kFSEventStreamEventFlagItemIsSymlink)
FLAG_PROPERTY(IsOwnEvent, kFSEventStreamEventFlagOwnEvent)
FLAG_PROPERTY(IsHardlink, kFSEventStreamEventFlagItemIsHardlink)
FLAG_PROPERTY(IsLastHardlink, kFSEventStreamEventFlagItemIsLastHardlink)
FLAG_PROPERTY(IsCloned, kFSEventStreamEventFlagItemCloned)

static int NativeEventInit(NativeEventObject *self, PyObject *args, PyObject *kwds)
{
Expand All @@ -157,11 +175,29 @@ static PyGetSetDef NativeEventProperties[] = {
{"flags", NativeEventTypeFlags, NULL, "The raw mask of flags as returend by FSEvents", NULL},
{"path", NativeEventTypePath, NULL, "The path for which this event was generated", NULL},
{"id", NativeEventTypeID, NULL, "The id of the generated event", NULL},
{"must_scan_subdirs", NativeEventTypeIsMustScanSubDirs, NULL, "True if application must rescan all subdirectories", NULL},
{"is_user_dropped", NativeEventTypeIsUserDropped, NULL, "True if a failure during event buffering occured", NULL},
{"is_kernel_dropped", NativeEventTypeIsKernelDropped, NULL, "True if a failure during event buffering occured", NULL},
{"is_event_ids_wrapped", NativeEventTypeIsEventIdsWrapped, NULL, "True if event_id wrapped around", NULL},
{"is_history_done", NativeEventTypeIsHistoryDone, NULL, "True if all historical events are done", NULL},
{"is_root_changed", NativeEventTypeIsRootChanged, NULL, "True if a change to one of the directories along the path to one of the directories you watch occurred", NULL},
{"is_mount", NativeEventTypeIsMount, NULL, "True if a volume is mounted underneath one of the paths being monitored", NULL},
{"is_unmount", NativeEventTypeIsUnmount, NULL, "True if a volume is unmounted underneath one of the paths being monitored", NULL},
{"is_created", NativeEventTypeIsCreated, NULL, "True if self.path was created on the filesystem", NULL},
{"is_removed", NativeEventTypeIsRemoved, NULL, "True if self.path was removed from the filesystem", NULL},
{"is_inode_meta_mod", NativeEventTypeIsInodeMetaMod, NULL, "True if meta data for self.path was modified ", NULL},
{"is_renamed", NativeEventTypeIsRenamed, NULL, "True if self.path was renamed on the filesystem", NULL},
{"is_modified", NativeEventTypeIsModified, NULL, "True if self.path was modified", NULL},
{"is_item_finder_info_modified", NativeEventTypeIsItemFinderInfoMod, NULL, "True if FinderInfo for self.path was modified", NULL},
{"is_owner_change", NativeEventTypeIsChangeOwner, NULL, "True if self.path had its ownership changed", NULL},
{"is_xattr_mod", NativeEventTypeIsXattrMod, NULL, "True if extended attributes for self.path were modified ", NULL},
{"is_file", NativeEventTypeIsFile, NULL, "True if self.path is a file", NULL},
{"is_directory", NativeEventTypeIsDirectory, NULL, "True if self.path is a directory", NULL},
{"is_symlink", NativeEventTypeIsSymlink, NULL, "True if self.path is a symbolic link", NULL},
{"is_own_event", NativeEventTypeIsOwnEvent, NULL, "True if the event originated from our own process", NULL},
{"is_hardlink", NativeEventTypeIsHardlink, NULL, "True if self.path is a hard link", NULL},
{"is_last_hardlink", NativeEventTypeIsLastHardlink, NULL, "True if self.path was the last hard link", NULL},
{"is_cloned", NativeEventTypeIsCloned, NULL, "True if self.path is a clone or was cloned", NULL},
{NULL, NULL, NULL, NULL, NULL},
};

Expand Down Expand Up @@ -256,9 +292,9 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref,
py_event_ids = PyList_New(num_events);
if (G_NOT(py_event_paths && py_event_flags && py_event_ids))
{
Py_DECREF(py_event_paths);
Py_DECREF(py_event_ids);
Py_DECREF(py_event_flags);
Py_XDECREF(py_event_paths);
Py_XDECREF(py_event_ids);
Py_XDECREF(py_event_flags);
return /*NULL*/;
}
for (i = 0; i < num_events; ++i)
Expand Down Expand Up @@ -303,6 +339,41 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref,
}


/**
* Converts a Python string object to an UTF-8 encoded ``CFStringRef``.
*
* :param py_string:
* A Python unicode or utf-8 encoded bytestring object.
* :returns:
* A new ``CFStringRef`` with the contents of ``py_string``, or ``NULL`` if an error occurred.
*/
CFStringRef PyString_AsUTF8EncodedCFStringRef(PyObject *py_string)
{
CFStringRef cf_string = NULL;

if (PyUnicode_Check(py_string)) {
PyObject* helper = PyUnicode_AsUTF8String(py_string);
if (!helper) {
return NULL;
}
cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(helper), kCFStringEncodingUTF8);
Py_DECREF(helper);
} else if (PyBytes_Check(py_string)) {
PyObject *utf8 = PyUnicode_FromEncodedObject(py_string, NULL, "strict");
if (!utf8) {
return NULL;
}
Py_DECREF(utf8);
cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(py_string), kCFStringEncodingUTF8);
} else {
PyErr_SetString(PyExc_TypeError, "Path to watch must be a string or a UTF-8 encoded bytes object.");
return NULL;
}

return cf_string;
}


/**
* Converts a list of Python strings to a ``CFMutableArray`` of
* UTF-8 encoded ``CFString`` instances and returns a pointer to
Expand All @@ -320,7 +391,6 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list)
{
Py_ssize_t i = 0;
Py_ssize_t string_list_size = 0;
const char *c_string = NULL;
CFMutableArrayRef array_of_cf_string = NULL;
CFStringRef cf_string = NULL;
PyObject *py_string = NULL;
Expand All @@ -340,14 +410,8 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list)
{
py_string = PyList_GetItem(py_string_list, i);
G_RETURN_NULL_IF_NULL(py_string);
if (PyUnicode_Check(py_string)) {
c_string = PyUnicode_AsUTF8(py_string);
} else {
c_string = PyBytes_AS_STRING(py_string);
}
cf_string = CFStringCreateWithCString(kCFAllocatorDefault,
c_string,
kCFStringEncodingUTF8);
cf_string = PyString_AsUTF8EncodedCFStringRef(py_string);
G_RETURN_NULL_IF_NULL(cf_string);
CFArraySetValueAtIndex(array_of_cf_string, i, cf_string);
CFRelease(cf_string);
}
Expand Down Expand Up @@ -454,7 +518,18 @@ watchdog_add_watch(PyObject *self, PyObject *args)
stream_ref = watchdog_FSEventStreamCreate(stream_callback_info_ref,
paths_to_watch,
(FSEventStreamCallback) &watchdog_FSEventStreamCallback);
if (!stream_ref) {
PyMem_Del(stream_callback_info_ref);
PyErr_SetString(PyExc_RuntimeError, "Failed creating fsevent stream");
return NULL;
}
value = PyCapsule_New(stream_ref, NULL, watchdog_pycapsule_destructor);
if (!value || !PyCapsule_IsValid(value, NULL)) {
PyMem_Del(stream_callback_info_ref);
FSEventStreamInvalidate(stream_ref);
FSEventStreamRelease(stream_ref);
return NULL;
}
PyDict_SetItem(watch_to_stream, watch, value);

/* Get a reference to the runloop for the emitter thread
Expand Down
47 changes: 45 additions & 2 deletions tests/test_fsevents.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
from os import mkdir, rmdir
from queue import Queue

from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
from watchdog.observers.api import ObservedWatch
from watchdog.observers.fsevents import FSEventsEmitter

from .shell import mkdtemp, rm
from .shell import mkdtemp, rm, touch

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -96,7 +97,49 @@ def on_thread_stop(self):
"""
a = p("a")
mkdir(a)
w = observer.schedule(event_queue, a, recursive=False)
w = observer.schedule(FileSystemEventHandler(), a, recursive=False)
rmdir(a)
time.sleep(0.1)
observer.unschedule(w)


def test_watchdog_recursive():
""" See https://github.com/gorakhargosh/watchdog/issues/706
"""
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import os.path

class Handler(FileSystemEventHandler):
def __init__(self):
FileSystemEventHandler.__init__(self)
self.changes = []

def on_any_event(self, event):
self.changes.append(os.path.basename(event.src_path))

handler = Handler()
observer = Observer()

watches = []
watches.append(observer.schedule(handler, str(p('')), recursive=True))

try:
observer.start()
time.sleep(0.1)

touch(p('my0.txt'))
mkdir(p('dir_rec'))
touch(p('dir_rec', 'my1.txt'))

expected = {"dir_rec", "my0.txt", "my1.txt"}
timeout_at = time.time() + 5
while not expected.issubset(handler.changes) and time.time() < timeout_at:
time.sleep(0.2)

assert expected.issubset(handler.changes), "Did not find expected changes. Found: {}".format(handler.changes)
finally:
for watch in watches:
observer.unschedule(watch)
observer.stop()
observer.join(1)

0 comments on commit 614ccfc

Please sign in to comment.