Skip to content

Commit

Permalink
Fixes #16 Caching the container contents for listDirectory()
Browse files Browse the repository at this point in the history 
Fixes #16 Caching the container contents for listDirectory()
  • Loading branch information
@jgulledge19
jgulledge19 committed Apr 19, 2017
1 parent 8441c26 commit 1c3df66
Showing 1 changed file with 120 additions and 8 deletions.
128 changes: 120 additions & 8 deletions core/components/awss3mediasource/model/awss3mediasource/awss3mediasource.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,17 @@

class AwsS3MediaSource extends modMediaSource implements modMediaSourceInterface
{
/** @var bool */
protected $use_cache = true;

/** @var array */
protected $cacheOptions = [
xPDO::OPT_CACHE_KEY => 'awss3mediasource'
];

/** @var int in seconds */
protected $cache_life = 3600;

/** @var Aws\S3\S3Client */
protected $driver;

Expand Down Expand Up @@ -44,6 +55,9 @@ public function initialize()
$this->xpdo->lexicon->load('core:source');
$this->properties = $this->getPropertyList();

$this->cache_life = (int)$this->xpdo->getOption('awss3mediasource.cache_life', $this->properties, 3600);
$this->use_cache = (bool)$this->xpdo->getOption('awss3mediasource.use_cache', $this->properties, true);

try {
$this->driver = new Aws\S3\S3Client([
'version' => 'latest',
Expand Down Expand Up @@ -168,7 +182,7 @@ public function getContainerList($path)
'type' => 'file',
'leaf' => true,
'path' => $currentPath,
'page' => $this->isBinary($encoded) ? $page : null,
'page' => $this->isBinary($encoded, false, $path) ? $page : null,
'pathRelative' => $url,
'directory' => $currentPath,
'url' => $url,
Expand Down Expand Up @@ -207,7 +221,12 @@ public function listDirectory($dir)
if (!empty($dir) && $dir != '/') {
$c['prefix'] = $dir;
}

if ( $this->use_cache ) {
$data = $this->getCache($dir);
if ($data) {
return $data;
}
}
try {
$result = $this->driver->listObjects([
'Bucket' => $this->bucket,
Expand Down Expand Up @@ -235,7 +254,9 @@ public function listDirectory($dir)
$files[] = $file['Key'];
}
}

if ( $this->use_cache ) {
$this->setCache($dir, [$files, $directories]);
}
return [$files, $directories];
}

Expand Down Expand Up @@ -315,24 +336,44 @@ public function getDirectoriesContextMenu()
*
* @param string $file
* @param boolean $isContent If the passed string in $file is actual file content
* @param boolean $path the full container path, needed to get cache
*
* @return boolean True if a binary file.
*/
public function isBinary($file, $isContent = false)
public function isBinary($file, $isContent = false, $path=false)
{
$cache = false;
if ($this->use_cache && $path !== false) {
$cache = $this->getCache($path);
if ($cache && isset($cache['isBinary'][$file])) {
return $cache['isBinary'][$file];
}
}
if (!$isContent) {
$fh = @fopen($file, 'r');
$blk = @fread($fh, 512);

@fclose($fh);
@clearstatcache();

return (substr_count($blk, "^ -~" /*. "^\r\n"*/) / 512 > 0.3) || (substr_count($blk, "\x00") > 0) ? false : true;
$binary = (substr_count($blk, "^ -~" /*. "^\r\n"*/) / 512 > 0.3) || (substr_count($blk, "\x00") > 0) ? false : true;

} else {
$content = str_replace(array("\n", "\r", "\t"), '', $file);

$binary = ctype_print($content) ? false : true;

}

$content = str_replace(array("\n", "\r", "\t"), '', $file);
if ( $this->use_cache && $path !== false && $cache !== false) {
if (!isset($cache['isBinary'])) {
$cache['isBinary'] = [];
}
$cache['isBinary'][$file] = $binary;
$this->setCache($path, $cache);
}

return ctype_print($content) ? false : true;
return $binary;
}

/**
Expand Down Expand Up @@ -446,7 +487,7 @@ public function getObjectsInContainer($path)
'pathname' => $url,
'pathRelative' => $currentPath,
'size' => 0,
'page' => $this->isBinary($encoded) ? $page : null,
'page' => $this->isBinary($encoded, false, $path) ? $page : null,
'leaf' => true,
);

Expand Down Expand Up @@ -564,6 +605,9 @@ public function createContainer($name, $parentContainer)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($parentContainer);
}
$this->xpdo->logManagerAction('directory_create', '', $newPath);

return true;
Expand Down Expand Up @@ -653,6 +697,12 @@ public function renameContainer($oldPath, $newName, $newNameFullPath=false, $del
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($oldPath);
// and the parent
$this->deleteCache(substr($oldPath, 0, strlen($oldPath) - (strlen(basename($oldPath))+1)));
}

return true;
}

Expand Down Expand Up @@ -758,6 +808,11 @@ public function removeContainer($path)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($path);
// and the parent
$this->deleteCache(substr($path, 0, strlen($path) - (strlen(basename($path))+1)));
}
$this->xpdo->logManagerAction('directory_remove', '', $path);

return true;
Expand Down Expand Up @@ -823,6 +878,10 @@ public function uploadObjectsToContainer($container, array $files = array())

$this->xpdo->log(xPDO::LOG_LEVEL_ERROR, '[AWS S3 MS] Error occurred when uploading file: ' . $e->getMessage());
}

if ( $this->use_cache ) {
$this->deleteCache($container);
}
}

if ($this->hasErrors() == true) return false;
Expand Down Expand Up @@ -1089,6 +1148,10 @@ public function createObject($path, $name, $content)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($path);
}

$this->xpdo->logManagerAction('file_create', '', $key);

return true;
Expand Down Expand Up @@ -1119,6 +1182,9 @@ public function updateObject($path, $content)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($path);
}
$this->xpdo->logManagerAction('file_update', '', $path);
return true;
}
Expand Down Expand Up @@ -1163,6 +1229,12 @@ public function renameObject($oldPath, $newName)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($oldPath);
// and the parent
$this->deleteCache(substr($oldPath, 0, strlen($oldPath) - (strlen(basename($oldPath))+1)));
}

$this->xpdo->logManagerAction('file_rename', '', $oldPath);

return true;
Expand Down Expand Up @@ -1239,6 +1311,12 @@ public function moveObject($from, $to, $point = 'append')
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($from);
// and the parent
$this->deleteCache(substr($from, 0, strlen($from) - (strlen(basename($from))+1)));
}

return true;
}

Expand Down Expand Up @@ -1284,6 +1362,12 @@ public function removeObject($path)
return false;
}

if ( $this->use_cache ) {
$this->deleteCache($path);
// and the parent
$this->deleteCache(substr($path, 0, strlen($path) - (strlen(basename($path))+1)));
}

$this->xpdo->logManagerAction('file_remove', '', $path);

return true;
Expand Down Expand Up @@ -1467,4 +1551,32 @@ public function getObjectUrl($object = '')

return $url . '/' . ltrim(str_replace($url, '', $object), '/');
}

/**
* @param string $key
*
* @return mixed
*/
protected function getCache($key)
{
return $this->xpdo->cacheManager->get($this->bucket.'.'.$key, $this->cacheOptions);
}

/**
* @param string $key
* @param mixed $data
*/
protected function setCache($key, $data)
{
$this->xpdo->cacheManager->set($this->bucket.'.'.$key, $data, $this->cache_life, $this->cacheOptions);
}

/**
* @param string $key
*/
protected function deleteCache($key)
{
$this->xpdo->cacheManager->delete($this->bucket.'.'.$key, $this->cacheOptions);
}

}

0 comments on commit 1c3df66

Please sign in to comment.