Correcting registry loading

This commit is contained in:
b-guild
2025-10-19 15:12:51 -07:00
parent fa610c7e77
commit a7504925f6
10 changed files with 262 additions and 168 deletions
+5 -1
View File
@@ -233,8 +233,12 @@ impl AssetItem {
fn can_be_dropped_to(&self, dest: &AssetItem) -> bool {
if self.path.is_file() && dest.path.is_dir() {
let Some(name) = self.path.file_name() else {
return false;
};
let dest_path = dest.path.join(name);
self.resource_manager.as_ref().is_some_and(|rm| {
block_on(rm.can_resource_be_moved(self.path.as_path(), dest.path.as_path(), true))
block_on(rm.can_resource_be_moved(self.path.as_path(), dest_path, true))
})
} else {
self.path.is_dir() && dest.path.is_dir()
+2 -4
View File
@@ -461,10 +461,8 @@ impl AssetItemContextMenu {
.send(Message::ShowInAssetBrowser(final_copy_path));
}
Err(err) => Log::err(format!(
"Failed to create a file for resource at path {}. \
Reason: {:?}",
final_copy_path.display(),
err
"Failed to create a file for resource at path {final_copy_path:?}. \
Reason: {err}",
)),
}
}
+12
View File
@@ -48,6 +48,18 @@ impl From<std::io::Error> for FileError {
}
}
impl From<&str> for FileError {
fn from(value: &str) -> Self {
Self::Custom(value.to_string())
}
}
impl From<String> for FileError {
fn from(value: String) -> Self {
Self::Custom(value)
}
}
#[cfg(target_os = "android")]
pub static ANDROID_APP: once_cell::sync::OnceCell<android_activity::AndroidApp> =
once_cell::sync::OnceCell::new();
+3
View File
@@ -311,6 +311,9 @@ pub fn hash_combine(lhs: u64, rhs: u64) -> u64 {
/// The last component of the path is permitted to not exist, so long as the rest of the path exists.
pub fn make_relative_path<P: AsRef<Path>>(path: P) -> Result<PathBuf, std::io::Error> {
let path = path.as_ref();
if path.as_os_str() == "." {
return Ok(path.to_path_buf());
}
// Canonicalization requires the full path to exist, so remove the file name before
// calling canonicalize.
let file_name = path.file_name().ok_or(std::io::Error::new(
+2 -11
View File
@@ -51,10 +51,7 @@ pub mod tilemap;
pub mod transform;
use crate::{
asset::{
self, io::ResourceIo, manager::ResourceManager, registry::ResourceRegistryStatus,
untyped::UntypedResource,
},
asset::{self, io::ResourceIo, manager::ResourceManager, untyped::UntypedResource},
core::{
algebra::Vector2,
color::Color,
@@ -346,14 +343,8 @@ impl SceneLoader {
serialization_context: Arc<SerializationContext>,
resource_manager: ResourceManager,
) -> Result<(Self, Vec<u8>), VisitError> {
let registry_status = resource_manager
.state()
.resource_registry
.safe_lock()
.status_flag();
// Wait until the registry is fully loaded.
let registry_status = registry_status.await;
if registry_status == ResourceRegistryStatus::Unknown {
if !resource_manager.registry_is_loaded().await {
return Err(VisitError::User(format!(
"Unable to load a scene from {} path, because the \
resource registry isn't loaded!",
+113 -12
View File
@@ -22,7 +22,9 @@
//! things such as loading assets within archive files
use fyrox_core::io::FileError;
use fyrox_core::{make_relative_path, replace_slashes};
use std::ffi::OsStr;
use std::path::Component;
use std::{
fmt::Debug,
fs::File,
@@ -66,6 +68,10 @@ impl FileReader for BufReader<File> {
/// Interface wrapping IO operations for doing this like loading files
/// for resources
pub trait ResourceIo: Send + Sync + 'static {
/// True if writing to files is possible through this object.
fn can_write(&self) -> bool;
/// True if reading the content of directories is possible through this object.
fn can_read_directories(&self) -> bool;
/// Attempts to load the file at the provided path returning
/// the entire byte contents of the file or an error
fn load_file<'a>(&'a self, path: &'a Path) -> ResourceIoFuture<'a, Result<Vec<u8>, FileError>>;
@@ -101,14 +107,13 @@ pub trait ResourceIo: Send + Sync + 'static {
dest: &'a Path,
) -> ResourceIoFuture<'a, Result<(), FileError>>;
/// Tries to convert the path to its canonical form (normalize it in other terms). This method
/// should guarantee correct behaviour for relative paths. Symlinks aren't mandatory to
/// Tries to convert the path to its canonical form (normalize it in other terms),
/// and put the path into a form that is suited for use in methods of this object.
/// Each file should have exactly one canonical path.
/// This method should guarantee correct behaviour for relative paths. Symlinks aren't mandatory to
/// follow.
fn canonicalize_path<'a>(
&'a self,
path: &'a Path,
) -> ResourceIoFuture<'a, Result<PathBuf, FileError>> {
Box::pin(ready(Ok(path.to_owned())))
fn canonicalize_path<'a>(&'a self, path: &'a Path) -> Result<PathBuf, FileError> {
Ok(path.to_owned())
}
/// Provides an iterator over the paths present in the provided
@@ -184,7 +189,50 @@ pub type PathIter = Box<dyn Iterator<Item = PathBuf>>;
#[cfg(not(target_arch = "wasm32"))]
pub type PathIter = Box<dyn Iterator<Item = PathBuf> + Send>;
/// Remove . and .. directories from a resource path, without accessing the file system,
/// and replace \ with /. There is no requirement that any part of the path actually exists.
/// The path "." is returned if the resulting path would otherwise be empty.
///
/// Because the file system is not accessed, all paths must be relative to the project root,
/// and this function will return an error if the path tries to go outside of it, such as by .. directories
/// or by being an absolute path.
pub fn normalize_path(path: impl AsRef<Path>) -> Result<PathBuf, FileError> {
let components = path.as_ref().components();
let mut ret = PathBuf::new();
for component in components {
match component {
Component::Prefix(..) | Component::RootDir => {
return Err(format!("Invalid path: {:?}", path.as_ref()).into());
}
Component::CurDir => {}
Component::ParentDir => {
if !ret.pop() {
panic!("Path may not start with ..");
}
}
Component::Normal(c) => {
ret.push(c);
}
}
}
if ret.as_os_str().is_empty() {
return Ok(".".into());
}
// The resource registry uses normalized paths with `/` slashes, and this step is needed
// mostly on Windows which uses `\` slashes.
Ok(replace_slashes(ret))
}
impl ResourceIo for FsResourceIo {
fn can_write(&self) -> bool {
cfg!(all(not(target_os = "android"), not(target_arch = "wasm32")))
}
fn can_read_directories(&self) -> bool {
cfg!(all(not(target_os = "android"), not(target_arch = "wasm32")))
}
fn load_file<'a>(&'a self, path: &'a Path) -> ResourceIoFuture<'a, Result<Vec<u8>, FileError>> {
Box::pin(fyrox_core::io::load_file(path))
}
@@ -241,11 +289,12 @@ impl ResourceIo for FsResourceIo {
})
}
fn canonicalize_path<'a>(
&'a self,
path: &'a Path,
) -> ResourceIoFuture<'a, Result<PathBuf, FileError>> {
Box::pin(async move { Ok(std::fs::canonicalize(path)?) })
fn canonicalize_path<'a>(&'a self, path: &'a Path) -> Result<PathBuf, FileError> {
if path.is_absolute() && self.can_read_directories() {
Ok(make_relative_path(path)?)
} else {
normalize_path(path)
}
}
/// wasm should fallback to the default no-op impl as im not sure if they
@@ -348,3 +397,55 @@ impl ResourceIo for FsResourceIo {
Box::pin(fyrox_core::io::is_dir(path))
}
}
#[cfg(test)]
mod test {
use super::*;
#[cfg(target_os = "windows")]
#[test]
fn test_normalize_backslash() {
let path = PathBuf::from("alpha\\beta\\..\\gamma");
assert_eq!(normalize_path(&path).unwrap().as_os_str(), "alpha/gamma");
}
#[cfg(target_os = "windows")]
#[test]
fn test_canonicalize_backslash() {
let rio = FsResourceIo;
let path = PathBuf::from("src\\test.txt");
assert_eq!(
rio.canonicalize_path(&path).unwrap().as_os_str(),
"src/test.txt"
);
}
#[test]
fn test_normalize() {
let path = PathBuf::from("alpha/beta");
assert_eq!(normalize_path(&path).unwrap().as_os_str(), "alpha/beta");
let path = PathBuf::from("alpha/..");
assert_eq!(normalize_path(&path).unwrap().as_os_str(), ".");
}
#[test]
fn test_canonicalize() {
let rio = FsResourceIo;
let path = PathBuf::from("src/test.txt");
assert_eq!(
rio.canonicalize_path(&path).unwrap().as_os_str(),
"src/test.txt"
);
let path = PathBuf::from("test.txt");
assert_eq!(
rio.canonicalize_path(&path).unwrap().as_os_str(),
"test.txt"
);
let path = PathBuf::from(".");
assert_eq!(rio.canonicalize_path(&path).unwrap().as_os_str(), ".");
let path = PathBuf::from("src")
.canonicalize()
.unwrap()
.join("test.txt");
assert_eq!(
rio.canonicalize_path(&path).unwrap().as_os_str(),
"src/test.txt"
);
}
}
+8 -2
View File
@@ -927,8 +927,14 @@ mod tests {
let res2 = resource_manager.request::<MyData>(path2);
assert_eq!(block_on(res1.clone()).unwrap().data_ref().data, 0);
assert_eq!(block_on(res2.clone()).unwrap().data_ref().data, 1);
let new_res1_path = ResourceRegistry::normalize_path(make_file_path(TEST_FOLDER3, 3));
let new_res2_path = ResourceRegistry::normalize_path(make_file_path(TEST_FOLDER3, 4));
let new_res1_path = resource_manager
.resource_io()
.canonicalize_path(&make_file_path(TEST_FOLDER3, 3))
.unwrap();
let new_res2_path = resource_manager
.resource_io()
.canonicalize_path(&make_file_path(TEST_FOLDER3, 4))
.unwrap();
block_on(resource_manager.move_resource(res1.as_ref(), &new_res1_path, true)).unwrap();
block_on(resource_manager.move_resource(res2.as_ref(), &new_res2_path, true)).unwrap();
assert_eq!(
+84 -50
View File
@@ -163,6 +163,7 @@ impl Display for ResourceRegistrationError {
}
/// All the required and validated data that is needed to move a resource from the path A to the path B.
#[derive(Debug)]
pub struct ResourceMoveContext {
relative_src_path: PathBuf,
relative_dest_path: PathBuf,
@@ -393,6 +394,19 @@ impl ResourceManager {
self.state.try_lock_for(timeout)
}
/// The resource manager keeps a registry of available resources, and this registry must be loaded
/// before any resources may be requested. This async method returns once the registry loading is complete
/// and it returns true if the loading was successful. It returns false if loading failed.
pub async fn registry_is_loaded(&self) -> bool {
let flag = self
.state()
.resource_registry
.safe_lock()
.status_flag()
.clone();
flag.await == ResourceRegistryStatus::Loaded
}
/// Returns the ResourceIo used by this resource manager
pub fn resource_io(&self) -> Arc<dyn ResourceIo> {
let state = self.state();
@@ -931,32 +945,41 @@ impl ResourceManagerState {
// Try to update the registry first.
#[cfg(not(any(target_arch = "wasm32", target_os = "android")))]
fyrox_core::futures::executor::block_on(async move {
let new_data =
ResourceRegistry::scan(resource_io.clone(), task_loaders, &path, excluded_folders)
.await;
let mut registry_lock = resource_registry.safe_lock();
registry_lock.modify().set_container(new_data);
registry_status.mark_as_loaded();
});
if resource_io.can_read_directories() && resource_io.can_write() {
fyrox_core::futures::executor::block_on(async move {
let new_data = ResourceRegistry::scan(
resource_io.clone(),
task_loaders,
&path,
excluded_folders,
)
.await;
let mut registry_lock = resource_registry.safe_lock();
registry_lock.modify().set_container(new_data);
registry_status.mark_as_loaded();
});
}
#[cfg(any(target_arch = "wasm32", target_os = "android"))]
self.task_pool.spawn_task(async move {
use crate::registry::RegistryContainerExt;
// Then load the registry.
info!("Trying to load or update the registry at {path:?}...");
match crate::registry::RegistryContainer::load_from_file(&path, &*resource_io).await {
Ok(registry) => {
let mut registry_lock = resource_registry.safe_lock();
registry_lock.modify().set_container(registry);
info!("Resource registry was loaded from {path:?} successfully!");
}
Err(error) => {
err!("Unable to load resource registry! Reason: {error}.");
}
};
registry_status.mark_as_loaded();
});
if !resource_io.can_read_directories() || !resource_io.can_write() {
self.task_pool.spawn_task(async move {
use crate::registry::RegistryContainerExt;
// Then load the registry.
info!("Trying to load or update the registry at {path:?}...");
match crate::registry::RegistryContainer::load_from_file(&path, &*resource_io).await
{
Ok(registry) => {
let mut registry_lock = resource_registry.safe_lock();
registry_lock.modify().set_container(registry);
info!("Resource registry was loaded from {path:?} successfully!");
}
Err(error) => {
err!("Unable to load resource registry! Reason: {error}.");
}
};
registry_status.mark_as_loaded();
});
}
}
/// Tries to find all the native resources registered in the resource registry, returns a collection
@@ -1402,15 +1425,21 @@ impl ResourceManagerState {
/// Searches the resource manager and the registry to find a resource with the given path,
/// including built-in resources. If no resource is found, a new UUID is generated and the
/// path is added to the registry and an unloaded resource is returned.
///
/// # Panics
///
/// Panics if the path is invalid, such as if it includes a directory that does not exist
/// or contains invalid characters.
pub fn find<P>(&mut self, path: P) -> UntypedResource
where
P: AsRef<Path>,
{
if let Some(built_in_resource) = self.built_in_resources.get(path.as_ref()) {
let path = path.as_ref();
if let Some(built_in_resource) = self.built_in_resources.get(path) {
return built_in_resource.resource.clone();
}
let path = ResourceRegistry::normalize_path(path);
let path = self.resource_io.canonicalize_path(path).unwrap();
if let Some(existing) = self.find_by_resource_path(&path) {
existing.clone()
@@ -1431,15 +1460,21 @@ impl ResourceManagerState {
}
/// Tries to load the resource at the given path.
///
/// # Panics
///
/// Panics if the path is invalid, such as if it includes a directory that does not exist
/// or contains invalid characters.
pub fn request<P>(&mut self, path: P) -> UntypedResource
where
P: AsRef<Path>,
{
if let Some(built_in_resource) = self.built_in_resources.get(path.as_ref()) {
let path = path.as_ref();
if let Some(built_in_resource) = self.built_in_resources.get(path) {
return built_in_resource.resource.clone();
}
let path = ResourceRegistry::normalize_path(path);
let path = self.resource_io.canonicalize_path(path).unwrap();
self.find_or_load(path)
}
@@ -1500,6 +1535,7 @@ impl ResourceManagerState {
self.task_pool.spawn_task(async move {
// Wait until the registry is fully loaded.
let registry_status = registry_status.await;
if registry_status == ResourceRegistryStatus::Unknown {
resource.commit_error(
PathBuf::default(),
@@ -1518,7 +1554,6 @@ impl ResourceManagerState {
in the registry!",
resource.resource_uuid(),
);
resource.commit_error(PathBuf::default(), error);
return;
};
@@ -1676,6 +1711,11 @@ impl ResourceManagerState {
/// to the file, otherwise the resource's path should already have been discovered by
/// [`ResourceManagerState::update_or_load_registry`].
/// If the manager already has a resource with this resource's UUID, return [`ResourceRegistrationError::AlreadyRegistered`].
///
/// # Panics
///
/// Panics if the path is invalid, such as if it includes a directory that does not exist
/// or contains invalid characters.
pub fn register(
&mut self,
resource: UntypedResource,
@@ -1685,7 +1725,7 @@ impl ResourceManagerState {
if self.find_by_uuid(resource_uuid).is_some() {
return Err(ResourceRegistrationError::AlreadyRegistered);
}
let path = ResourceRegistry::normalize_path(path);
let path = self.resource_io.canonicalize_path(path.as_ref()).unwrap();
{
let mut header = resource.lock();
header.kind.make_external();
@@ -1774,8 +1814,8 @@ impl ResourceManagerState {
let src_path = src_path.as_ref();
let dest_path = dest_path.as_ref();
let relative_src_path = fyrox_core::make_relative_path(src_path)?;
let relative_dest_path = fyrox_core::make_relative_path(dest_path)?;
let relative_src_path = self.resource_io.canonicalize_path(src_path)?;
let relative_dest_path = self.resource_io.canonicalize_path(dest_path)?;
if let Some(file_stem) = relative_dest_path.file_stem() {
if !self.resource_io.is_valid_file_name(file_stem) {
@@ -1794,8 +1834,8 @@ impl ResourceManagerState {
}
let registry_lock_guard = self.resource_registry.safe_lock();
let absolute_registry_dir = if let Some(directory) = registry_lock_guard.directory() {
fyrox_core::replace_slashes(self.resource_io.canonicalize_path(directory).await?)
let registry_dir = if let Some(directory) = registry_lock_guard.directory() {
self.resource_io.canonicalize_path(directory)?
} else {
return Err(ResourceMovementError::ResourceRegistryLocationUnknown {
resource_path: relative_src_path.clone(),
@@ -1807,23 +1847,17 @@ impl ResourceManagerState {
resource_path: relative_src_path.clone(),
})?;
let relative_dest_dir = relative_dest_path.parent().unwrap_or(Path::new("."));
let absolute_dest_dir = fyrox_core::replace_slashes(
self.resource_io
.canonicalize_path(relative_dest_dir)
.await?,
);
let absolute_src_path = fyrox_core::replace_slashes(
self.resource_io
.canonicalize_path(&relative_src_path)
.await?,
);
if !absolute_dest_dir.starts_with(&absolute_registry_dir) {
let Some(relative_dest_dir) = relative_dest_path.parent() else {
return Err(ResourceMovementError::DestinationPathIsInvalid {
src_path: relative_src_path.clone(),
dest_path: relative_dest_path.clone(),
});
};
if !relative_dest_dir.starts_with(&registry_dir) {
return Err(ResourceMovementError::OutsideOfRegistry {
absolute_src_path,
absolute_dest_dir,
absolute_registry_dir,
absolute_src_path: relative_src_path,
absolute_dest_dir: relative_dest_dir.to_path_buf(),
absolute_registry_dir: registry_dir,
});
}
+26 -86
View File
@@ -23,8 +23,7 @@
use crate::{
core::{
append_extension, err, info, io::FileError, ok_or_return, parking_lot::Mutex,
replace_slashes, warn, Uuid,
append_extension, err, info, io::FileError, ok_or_return, parking_lot::Mutex, warn, Uuid,
},
io::ResourceIo,
loader::ResourceLoadersContainer,
@@ -37,7 +36,7 @@ use ron::ser::PrettyConfig;
use std::{
collections::BTreeMap,
future::Future,
path::{Component, Path, PathBuf},
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
task::{Context, Poll},
@@ -144,6 +143,7 @@ impl ResourceRegistryStatusFlag {
/// Marks the registry as loaded.
pub fn mark_as_loaded(&self) {
self.mark_as(ResourceRegistryStatus::Loaded);
info!("Resource registry finished loading.");
}
/// Marks the registry as unknown, due to an error.
@@ -175,47 +175,6 @@ impl Future for ResourceRegistryStatusFlag {
}
}
async fn make_relative_path_async<P: AsRef<Path>>(
path: P,
io: &dyn ResourceIo,
) -> Result<PathBuf, FileError> {
let path = path.as_ref();
// Canonicalization requires the full path to exist, so remove the file name before
// calling canonicalize.
let file_name = path.file_name().ok_or(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("Invalid path: {}", path.display()),
))?;
let dir = path.parent();
let dir = if let Some(dir) = dir {
if dir.as_os_str().is_empty() {
Path::new(".")
} else {
dir
}
} else {
Path::new(".")
};
let canon_path = io
.canonicalize_path(dir)
.await
.map_err(|err| {
FileError::Custom(format!(
"Unable to canonicalize '{}'. Reason: {err:?}",
dir.display()
))
})?
.join(file_name);
let current_dir = io.canonicalize_path(&std::env::current_dir()?).await?;
match canon_path.strip_prefix(current_dir) {
Ok(relative_path) => Ok(replace_slashes(relative_path)),
Err(err) => Err(FileError::Custom(format!(
"unable to strip prefix from '{}'! Reason: {err}",
canon_path.display()
))),
}
}
/// A mutable reference to a resource registry. Automatically saves the registry back to a source
/// from which it was loaded when an instance of this object is dropped. To prevent saving use
/// [`std::mem::forget`] after you've finished working with the mutable reference.
@@ -402,36 +361,15 @@ impl ResourceRegistry {
self.status.clone()
}
/// Normalizes the path by resolving all `.` and `..` and removing any prefixes. Also replaces
/// `\\` slashes to cross-platform `/` slashes.
pub fn normalize_path(path: impl AsRef<Path>) -> PathBuf {
let mut components = path.as_ref().components().peekable();
let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
components.next();
PathBuf::from(c.as_os_str())
} else {
PathBuf::new()
};
for component in components {
match component {
Component::Prefix(..) => unreachable!(),
Component::RootDir => {
ret.push(component.as_os_str());
}
Component::CurDir => {}
Component::ParentDir => {
ret.pop();
}
Component::Normal(c) => {
ret.push(c);
}
}
}
// The resource registry uses normalized paths with `/` slashes, and this step is needed
// mostly on Windows which uses `\` slashes.
replace_slashes(ret)
/// Normalizes the path by resolving all `.` and `..` and removing any prefixes.
/// Absolute paths are converted to relative paths by removing the project root prefix, if possible.
/// Also replaces `\\` slashes to cross-platform `/` slashes.
///
/// # Panics
///
/// Panics if the given path is invalid, such as if it includes a directory that does not exist.
pub fn normalize_path(&self, path: impl AsRef<Path>) -> PathBuf {
self.io.canonicalize_path(path.as_ref()).unwrap()
}
/// Returns a reference to the actual container of the resource entries.
@@ -456,16 +394,18 @@ impl ResourceRegistry {
/// Asynchronously saves the registry.
pub async fn save(&self) {
match self.paths.save(&self.path, &*self.io).await {
Err(error) => {
err!(
"Unable to write the resource registry at the {:?} path! Reason: {:?}",
self.path,
error
)
}
Ok(_) => {
info!("The registry was successfully saved to {:?}!", self.path)
if self.io.can_write() {
match self.paths.save(&self.path, &*self.io).await {
Err(error) => {
err!(
"Unable to write the resource registry at the {:?} path! Reason: {:?}",
self.path,
error
)
}
Ok(_) => {
info!("The registry was successfully saved to {:?}!", self.path)
}
}
}
}
@@ -473,7 +413,7 @@ impl ResourceRegistry {
/// Same as [`Self::save`], but synchronous.
pub fn save_sync(&self) {
#[cfg(not(target_arch = "wasm32"))]
{
if self.io.can_write() {
match self.paths.save_sync(&self.path, &*self.io) {
Err(error) => {
err!(
@@ -554,7 +494,7 @@ impl ResourceRegistry {
.collect::<Vec<_>>();
while let Some(fs_path) = paths_to_visit.pop() {
let path = match make_relative_path_async(&fs_path, &*resource_io).await {
let path = match resource_io.canonicalize_path(&fs_path) {
Ok(path) => path,
Err(err) => {
warn!(
+7 -2
View File
@@ -322,8 +322,10 @@ use crate::{
use copypasta::ClipboardContext;
use fxhash::{FxHashMap, FxHashSet};
use fyrox_resource::{
io::FsResourceIo, io::ResourceIo, manager::ResourceManager, untyped::UntypedResource, Resource,
ResourceData,
io::{FsResourceIo, ResourceIo},
manager::ResourceManager,
untyped::UntypedResource,
Resource, ResourceData,
};
use serde::{Deserialize, Serialize};
use std::{
@@ -3270,6 +3272,9 @@ impl UserInterface {
resource_manager: ResourceManager,
io: &dyn ResourceIo,
) -> Result<Self, VisitError> {
if !resource_manager.registry_is_loaded().await {
return Err("The resource registry is unavailable!".to_string().into());
}
let mut ui = {
let mut visitor = Visitor::load_from_memory(&io.load_file(path.as_ref()).await?)?;
let (sender, receiver) = mpsc::channel();