Skip to content

Commit

Permalink
refactor: Store complete inside bits and add more examples (#1397)
Browse files Browse the repository at this point in the history 
* refactor: Store complete inside bits

Signed-off-by: Xuanwo <[email protected]>

* Add examples

Signed-off-by: Xuanwo <[email protected]>

---------

Signed-off-by: Xuanwo <[email protected]>
  • Loading branch information
@Xuanwo
Xuanwo authored Feb 22, 2023
1 parent 1dc5521 commit 7349c90
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 42 deletions.
19 changes: 12 additions & 7 deletions src/layers/complete.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,16 +292,21 @@ impl<A: Accessor> LayeredAccessor for CompleteReaderAccessor<A> {
}

async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
self.inner
.stat(path, args)
.await
.map(|v| v.map_metadata(|m| m.with_complete()))
self.inner.stat(path, args).await.map(|v| {
v.map_metadata(|m| {
let bit = m.bit();
m.with_bit(bit | ObjectMetadataKey::Complete)
})
})
}

fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
self.inner
.blocking_stat(path, args)
.map(|v| v.map_metadata(|m| m.with_complete()))
self.inner.blocking_stat(path, args).map(|v| {
v.map_metadata(|m| {
let bit = m.bit();
m.with_bit(bit | ObjectMetadataKey::Complete)
})
})
}

async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
Expand Down
35 changes: 18 additions & 17 deletions src/object/metadata.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ use crate::*;
/// a.k.a., `Entry`'s content length could be `None`.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct ObjectMetadata {
/// Mark if this metadata is complete or not.
complete: bool,
/// bit stores current key store.
bit: FlagSet<ObjectMetadataKey>,

Expand All @@ -55,11 +53,15 @@ pub struct ObjectMetadata {
impl ObjectMetadata {
/// Create a new object metadata
pub fn new(mode: ObjectMode) -> Self {
Self {
// If mode is dir, we will set complete to true.
complete: mode == ObjectMode::DIR,
bit: ObjectMetadataKey::Mode.into(),
// Mode is required to be set for object metadata.
let mut bit = ObjectMetadataKey::Mode.into();
// If object mode is dir, we should always mark it as complete.
if mode == ObjectMode::DIR {
bit |= ObjectMetadataKey::Complete
}

Self {
bit,
mode,

content_length: None,
Expand All @@ -72,22 +74,17 @@ impl ObjectMetadata {
}
}

/// If this object metadata if complete
pub(crate) fn is_complete(&self) -> bool {
self.complete
}

/// Make this object metadata if complete.
pub(crate) fn with_complete(mut self) -> Self {
self.complete = true;
self
}

/// Get the bit from object metadata.
pub(crate) fn bit(&self) -> FlagSet<ObjectMetadataKey> {
self.bit
}

/// Set bit with given.
pub(crate) fn with_bit(mut self, bit: impl Into<FlagSet<ObjectMetadataKey>>) -> Self {
self.bit = bit.into();
self
}

/// Object mode represent this object's mode.
pub fn mode(&self) -> ObjectMode {
self.mode
Expand Down Expand Up @@ -358,6 +355,10 @@ flags! {
/// the meta has been stored, we will return directly. If no, we will
/// call `stat` internally to fecth the metadata.
pub enum ObjectMetadataKey: u64 {
/// The special object metadata key that used to mark this object
/// already contains all metadata.
Complete,

/// Key for mode.
Mode,
/// Key for content disposition.
Expand Down
109 changes: 91 additions & 18 deletions src/object/object.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1237,18 +1237,39 @@ impl Object {
///
/// # Examples
///
/// ## Query already cached metadata
///
/// By query metadata with `None`, we can only query in-memory metadata
/// cache. In this way, we can make sure that no API call will send.
///
/// ```
/// # use anyhow::Result;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// #
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
/// let meta = op.object("test").metadata(None).await?;
/// let _ = meta.content_length();
/// let _ = meta.content_type();
/// # Ok(())
/// # }
/// ```
///
/// ## Query content length and content type
///
/// ```
/// # use anyhow::Result;
/// # use futures::io;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// use opendal::ObjectMetadataKey;
/// #
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
/// let meta = op
/// .object("test")
/// .metadata({
/// use opendal::ObjectMetadataKey::*;
/// use ObjectMetadataKey::*;
/// ContentLength | ContentType
/// })
/// .await?;
Expand All @@ -1257,15 +1278,34 @@ impl Object {
/// # Ok(())
/// # }
/// ```
///
/// ## Query all metadata
///
/// By query metadata with `Complete`, we can make sure that we have fetched all metadata of this object.
///
/// ```
/// # use anyhow::Result;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// use opendal::ObjectMetadataKey;
/// #
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
/// let meta = op
/// .object("test")
/// .metadata({ ObjectMetadataKey::Complete })
/// .await?;
/// let _ = meta.content_length();
/// let _ = meta.content_type();
/// # Ok(())
/// # }
/// ```
pub async fn metadata(
&mut self,
flags: impl Into<FlagSet<ObjectMetadataKey>>,
) -> Result<Arc<ObjectMetadata>> {
if let Some(meta) = &self.meta {
if meta.is_complete() {
return Ok(meta.clone());
}
if meta.bit().contains(flags) {
if meta.bit().contains(flags) || meta.bit().contains(ObjectMetadataKey::Complete) {
return Ok(meta.clone());
}
}
Expand Down Expand Up @@ -1302,21 +1342,57 @@ impl Object {
///
/// # Examples
///
/// ## Query already cached metadata
///
/// By query metadata with `None`, we can only query in-memory metadata
/// cache. In this way, we can make sure that no API call will send.
///
/// ```
/// # use anyhow::Result;
/// # use futures::io;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// #
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
/// # fn test(op: Operator) -> Result<()> {
/// let meta = op.object("test").blocking_metadata(None)?;
/// let _ = meta.content_length();
/// let _ = meta.content_type();
/// # Ok(())
/// # }
/// ```
///
/// ## Query content length and content type
///
/// ```
/// # use anyhow::Result;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// use opendal::ObjectMetadataKey;
/// #
/// # fn test(op: Operator) -> Result<()> {
/// let meta = op.object("test").blocking_metadata({
/// use ObjectMetadataKey::*;
/// ContentLength | ContentType
/// })?;
/// let _ = meta.content_length();
/// let _ = meta.content_type();
/// # Ok(())
/// # }
/// ```
///
/// ## Query all metadata
///
/// By query metadata with `Complete`, we can make sure that we have fetched all metadata of this object.
///
/// ```
/// # use anyhow::Result;
/// # use opendal::Operator;
/// use opendal::ErrorKind;
/// use opendal::ObjectMetadataKey;
/// #
/// # fn test(op: Operator) -> Result<()> {
/// let meta = op
/// .object("test")
/// .metadata({
/// use opendal::ObjectMetadataKey::*;
/// ContentLength | ContentType
/// })
/// .await?;
/// .blocking_metadata({ ObjectMetadataKey::Complete })?;
/// let _ = meta.content_length();
/// let _ = meta.content_type();
/// # Ok(())
Expand All @@ -1327,10 +1403,7 @@ impl Object {
flags: impl Into<FlagSet<ObjectMetadataKey>>,
) -> Result<Arc<ObjectMetadata>> {
if let Some(meta) = &self.meta {
if meta.is_complete() {
return Ok(meta.clone());
}
if meta.bit().contains(flags) {
if meta.bit().contains(flags) || meta.bit().contains(ObjectMetadataKey::Complete) {
return Ok(meta.clone());
}
}
Expand Down

1 comment on commit 7349c90

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for opendal ready!

✅ Preview
https://opendal-67dyn7zrv-databend.vercel.app

Built with commit 7349c90.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.