Skip to content

Commit

Permalink
leveldb: Win32WritableFile without memory mapping
Browse files Browse the repository at this point in the history 
Use a straightforward Win32WritableFile for writing files, equivalent to
PosixWritableFile in posix_env.cc, but using only the Win32 API.

The goal is to solve leveldb corruption issues on windows after crashes, which
are frequently reported.
  • Loading branch information
@laanwj
laanwj committed Nov 4, 2015
1 parent 7d41e6f commit 7aa105e
Showing 1 changed file with 39 additions and 197 deletions.
236 changes: 39 additions & 197 deletions util/env_win.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,39 +103,20 @@ class Win32RandomAccessFile : public RandomAccessFile
DISALLOW_COPY_AND_ASSIGN(Win32RandomAccessFile);
};

class Win32MapFile : public WritableFile
class Win32WritableFile : public WritableFile
{
public:
Win32MapFile(const std::string& fname);
Win32WritableFile(const std::string& fname);
~Win32WritableFile();

~Win32MapFile();
virtual Status Append(const Slice& data);
virtual Status Close();
virtual Status Flush();
virtual Status Sync();
BOOL isEnable();
private:
std::string _filename;
HANDLE _hFile;
size_t _page_size;
size_t _map_size; // How much extra memory to map at a time
char* _base; // The mapped region
HANDLE _base_handle;
char* _limit; // Limit of the mapped region
char* _dst; // Where to write next (in range [base_,limit_])
char* _last_sync; // Where have we synced up to
uint64_t _file_offset; // Offset of base_ in file
//LARGE_INTEGER file_offset_;
// Have we done an munmap of unsynced data?
bool _pending_sync;

// Roundup x to a multiple of y
static size_t _Roundup(size_t x, size_t y);
size_t _TruncateToPageBoundary(size_t s);
bool _UnmapCurrentRegion();
bool _MapNewRegion();
DISALLOW_COPY_AND_ASSIGN(Win32MapFile);
BOOL _Init(LPCWSTR Path);
std::string filename_;
::HANDLE _hFile;
};

class Win32FileLock : public FileLock
Expand Down Expand Up @@ -442,202 +423,63 @@ void Win32RandomAccessFile::_CleanUp()
}
}

size_t Win32MapFile::_Roundup( size_t x, size_t y )
Win32WritableFile::Win32WritableFile(const std::string& fname)
: filename_(fname)
{
return ((x + y - 1) / y) * y;
std::wstring path;
ToWidePath(fname, path);
DWORD Flag = PathFileExistsW(path.c_str()) ? OPEN_EXISTING : CREATE_ALWAYS;
_hFile = CreateFileW(path.c_str(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE,
NULL,
Flag,
FILE_ATTRIBUTE_NORMAL,
NULL);
// CreateFileW returns INVALID_HANDLE_VALUE in case of error, always check isEnable() before use
}

size_t Win32MapFile::_TruncateToPageBoundary( size_t s )
Win32WritableFile::~Win32WritableFile()
{
s -= (s & (_page_size - 1));
assert((s % _page_size) == 0);
return s;
if (_hFile != INVALID_HANDLE_VALUE)
Close();
}

bool Win32MapFile::_UnmapCurrentRegion()
Status Win32WritableFile::Append(const Slice& data)
{
bool result = true;
if (_base != NULL) {
if (_last_sync < _limit) {
// Defer syncing this data until next Sync() call, if any
_pending_sync = true;
}
if (!UnmapViewOfFile(_base) || !CloseHandle(_base_handle))
result = false;
_file_offset += _limit - _base;
_base = NULL;
_base_handle = NULL;
_limit = NULL;
_last_sync = NULL;
_dst = NULL;
// Increase the amount we map the next time, but capped at 1MB
if (_map_size < (1<<20)) {
_map_size *= 2;
}
DWORD r = 0;
if (!WriteFile(_hFile, data.data(), data.size(), &r, NULL) || r != data.size()) {
return Status::IOError("Win32WritableFile.Append::WriteFile: "+filename_, Win32::GetLastErrSz());
}
return result;
}

bool Win32MapFile::_MapNewRegion()
{
assert(_base == NULL);
//LONG newSizeHigh = (LONG)((file_offset_ + map_size_) >> 32);
//LONG newSizeLow = (LONG)((file_offset_ + map_size_) & 0xFFFFFFFF);
DWORD off_hi = (DWORD)(_file_offset >> 32);
DWORD off_lo = (DWORD)(_file_offset & 0xFFFFFFFF);
LARGE_INTEGER newSize;
newSize.QuadPart = _file_offset + _map_size;
SetFilePointerEx(_hFile, newSize, NULL, FILE_BEGIN);
SetEndOfFile(_hFile);

_base_handle = CreateFileMappingA(
_hFile,
NULL,
PAGE_READWRITE,
0,
0,
0);
if (_base_handle != NULL) {
_base = (char*) MapViewOfFile(_base_handle,
FILE_MAP_ALL_ACCESS,
off_hi,
off_lo,
_map_size);
if (_base != NULL) {
_limit = _base + _map_size;
_dst = _base;
_last_sync = _base;
return true;
}
}
return false;
return Status::OK();
}

Win32MapFile::Win32MapFile( const std::string& fname) :
_filename(fname),
_hFile(NULL),
_page_size(Win32::g_PageSize),
_map_size(_Roundup(65536, Win32::g_PageSize)),
_base(NULL),
_base_handle(NULL),
_limit(NULL),
_dst(NULL),
_last_sync(NULL),
_file_offset(0),
_pending_sync(false)
Status Win32WritableFile::Close()
{
std::wstring path;
ToWidePath(fname, path);
_Init(path.c_str());
assert((Win32::g_PageSize & (Win32::g_PageSize - 1)) == 0);
}

Status Win32MapFile::Append( const Slice& data )
{
const char* src = data.data();
size_t left = data.size();
Status s;
while (left > 0) {
assert(_base <= _dst);
assert(_dst <= _limit);
size_t avail = _limit - _dst;
if (avail == 0) {
if (!_UnmapCurrentRegion() ||
!_MapNewRegion()) {
return Status::IOError("WinMmapFile.Append::UnmapCurrentRegion or MapNewRegion: ", Win32::GetLastErrSz());
}
}
size_t n = (left <= avail) ? left : avail;
memcpy(_dst, src, n);
_dst += n;
src += n;
left -= n;
}
return s;
}

Status Win32MapFile::Close()
{
Status s;
size_t unused = _limit - _dst;
if (!_UnmapCurrentRegion()) {
s = Status::IOError("WinMmapFile.Close::UnmapCurrentRegion: ",Win32::GetLastErrSz());
} else if (unused > 0) {
// Trim the extra space at the end of the file
LARGE_INTEGER newSize;
newSize.QuadPart = _file_offset - unused;
if (!SetFilePointerEx(_hFile, newSize, NULL, FILE_BEGIN)) {
s = Status::IOError("WinMmapFile.Close::SetFilePointer: ",Win32::GetLastErrSz());
} else
SetEndOfFile(_hFile);
}
if (!CloseHandle(_hFile)) {
if (s.ok()) {
s = Status::IOError("WinMmapFile.Close::CloseHandle: ", Win32::GetLastErrSz());
}
return Status::IOError("Win32WritableFile.Close::CloseHandle: "+filename_, Win32::GetLastErrSz());
}
_hFile = INVALID_HANDLE_VALUE;
_base = NULL;
_base_handle = NULL;
_limit = NULL;

return s;
}

Status Win32MapFile::Sync()
{
Status s;
if (_pending_sync) {
// Some unmapped data was not synced
_pending_sync = false;
if (!FlushFileBuffers(_hFile)) {
s = Status::IOError("WinMmapFile.Sync::FlushFileBuffers: ",Win32::GetLastErrSz());
}
}
if (_dst > _last_sync) {
// Find the beginnings of the pages that contain the first and last
// bytes to be synced.
size_t p1 = _TruncateToPageBoundary(_last_sync - _base);
size_t p2 = _TruncateToPageBoundary(_dst - _base - 1);
_last_sync = _dst;
if (!FlushViewOfFile(_base + p1, p2 - p1 + _page_size)) {
s = Status::IOError("WinMmapFile.Sync::FlushViewOfFile: ",Win32::GetLastErrSz());
}
}
return s;
return Status::OK();
}

Status Win32MapFile::Flush()
Status Win32WritableFile::Flush()
{
// Nothing to do here, there are no application-side buffers
return Status::OK();
}

Win32MapFile::~Win32MapFile()
Status Win32WritableFile::Sync()
{
if (_hFile != INVALID_HANDLE_VALUE) {
Win32MapFile::Close();
if (!FlushFileBuffers(_hFile)) {
return Status::IOError("Win32WritableFile.Sync::FlushFileBuffers "+filename_, Win32::GetLastErrSz());
}
return Status::OK();
}

BOOL Win32MapFile::_Init( LPCWSTR Path )
{
DWORD Flag = PathFileExistsW(Path) ? OPEN_EXISTING : CREATE_ALWAYS;
_hFile = CreateFileW(Path,
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE,
NULL,
Flag,
FILE_ATTRIBUTE_NORMAL,
NULL);
if(!_hFile || _hFile == INVALID_HANDLE_VALUE)
return FALSE;
else
return TRUE;
}

BOOL Win32MapFile::isEnable()
BOOL Win32WritableFile::isEnable()
{
return _hFile ? TRUE : FALSE;
return _hFile != INVALID_HANDLE_VALUE;
}

Win32FileLock::Win32FileLock( const std::string& fname ) :
Expand Down Expand Up @@ -981,7 +823,7 @@ Status Win32Env::NewLogger( const std::string& fname, Logger** result )
{
Status sRet;
std::string path = fname;
Win32MapFile* pMapFile = new Win32MapFile(ModifyPath(path));
Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path));
if(!pMapFile->isEnable()){
delete pMapFile;
*result = NULL;
Expand All @@ -995,7 +837,7 @@ Status Win32Env::NewWritableFile( const std::string& fname, WritableFile** resul
{
Status sRet;
std::string path = fname;
Win32MapFile* pFile = new Win32MapFile(ModifyPath(path));
Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path));
if(!pFile->isEnable()){
*result = NULL;
sRet = Status::IOError(fname,Win32::GetLastErrSz());
Expand Down

0 comments on commit 7aa105e

Please sign in to comment.