From 42009a353ad5c87500fae8324386b54f2f468f62 Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:14:53 +0100 Subject: [PATCH 1/2] fix spritesheet gen on linux --- src/iconforge/byond.rs | 5 ++ src/iconforge/image_cache.rs | 85 +++++++++++-------- src/iconforge/spritesheet.rs | 156 +++++++++++++++++++++-------------- tests/iconforge/mod.rs | 2 +- 4 files changed, 152 insertions(+), 96 deletions(-) diff --git a/src/iconforge/byond.rs b/src/iconforge/byond.rs index fefe4ad4..d5c6fee4 100644 --- a/src/iconforge/byond.rs +++ b/src/iconforge/byond.rs @@ -69,6 +69,11 @@ byond_fn!(fn iconforge_check(id) { byond_fn!( fn iconforge_cleanup() { + // Only perform cleanup if no jobs are currently using the icon cache + if image_cache::CACHE_ACTIVE.load(std::sync::atomic::Ordering::SeqCst) > 0 { + return Some("Skipped, cache in use"); + } + image_cache::icon_cache_clear(); image_cache::image_cache_clear(); Some("Ok") diff --git a/src/iconforge/image_cache.rs b/src/iconforge/image_cache.rs index 890542c6..93598f52 100644 --- a/src/iconforge/image_cache.rs +++ b/src/iconforge/image_cache.rs @@ -5,11 +5,29 @@ use dmi::{ icon::{Icon, IconState, dir_to_dmi_index}, }; use image::RgbaImage; -use once_cell::sync::Lazy; -use std::{fs::File, hash::BuildHasherDefault, io::BufReader, sync::Arc}; +use once_cell::sync::{Lazy, OnceCell}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{fs::File, hash::BuildHasherDefault, io::BufReader, path::PathBuf, sync::Arc}; use tracy_full::zone; use twox_hash::XxHash64; +pub static CACHE_ACTIVE: AtomicUsize = AtomicUsize::new(0); + +struct CacheGuard; + +impl CacheGuard { + fn new() -> Self { + CACHE_ACTIVE.fetch_add(1, Ordering::SeqCst); + CacheGuard + } +} + +impl Drop for CacheGuard { + fn drop(&mut self) { + CACHE_ACTIVE.fetch_sub(1, Ordering::SeqCst); + } +} + /// A cache of UniversalIcon to UniversalIconData. In order for something to exist in this cache, it must have had any transforms applied to the images. static ICON_STATES: Lazy< DashMap, BuildHasherDefault>, @@ -20,6 +38,7 @@ static ICON_STATES_FLAT: Lazy< > = Lazy::new(|| DashMap::with_hasher(BuildHasherDefault::::default())); pub fn image_cache_contains(icon: &UniversalIcon, flatten: bool) -> bool { + let _guard = CacheGuard::new(); if flatten { ICON_STATES_FLAT.contains_key(icon) } else { @@ -28,6 +47,7 @@ pub fn image_cache_contains(icon: &UniversalIcon, flatten: bool) -> bool { } pub fn image_cache_clear() { + let _guard = CacheGuard::new(); ICON_STATES.clear(); ICON_STATES_FLAT.clear(); } @@ -44,6 +64,7 @@ impl UniversalIcon { flatten: bool, ) -> Result<(Arc, bool), String> { zone!("universal_icon_to_image_data"); + let _guard = CacheGuard::new(); if cached { zone!("check_image_cache"); if let Some(entry) = if flatten { @@ -186,6 +207,7 @@ pub fn cache_transformed_images( flatten: bool, ) { zone!("cache_transformed_images"); + let _guard = CacheGuard::new(); if flatten { ICON_STATES_FLAT.insert(uni_icon.to_owned(), image_data.to_owned()); } else { @@ -194,47 +216,44 @@ pub fn cache_transformed_images( } /* ---- DMI CACHING ---- */ +type IconMap = DashMap>, BuildHasherDefault>; /// A cache of DMI filepath -> Icon objects. -static ICON_FILES: Lazy, BuildHasherDefault>> = +static ICON_FILES: Lazy = Lazy::new(|| DashMap::with_hasher(BuildHasherDefault::::default())); pub fn icon_cache_clear() { + let _guard = CacheGuard::new(); ICON_FILES.clear(); } +pub static ICON_ROOT: Lazy = Lazy::new(|| std::env::current_dir().unwrap()); + /// Given a DMI filepath, returns a DMI Icon structure and caches it. pub fn filepath_to_dmi(icon_path: &str) -> Result, String> { zone!("filepath_to_dmi"); - { - zone!("check_dmi_exists"); - if let Some(found) = ICON_FILES.get(icon_path) { - return Ok(found.clone()); - } - } - let icon_file = match File::open(icon_path) { - Ok(icon_file) => icon_file, - Err(err) => { - return Err(format!("Failed to open DMI '{icon_path}' - {err}")); - } - }; - let reader = BufReader::new(icon_file); - let dmi: Icon; - { + + let full_path = ICON_ROOT.join(icon_path); + + let cell = ICON_FILES.entry(icon_path.to_owned()).or_default(); + + cell.get_or_try_init(|| { + zone!("open_dmi_file"); + let icon_file = File::open(&full_path).map_err(|err| { + format!( + "Failed to open DMI '{}' (resolved to '{}') - {}", + icon_path, + full_path.display(), + err + ) + })?; + + let reader = BufReader::new(icon_file); + zone!("parse_dmi"); - dmi = match Icon::load(reader) { - Ok(dmi) => dmi, - Err(err) => { - return Err(format!("DMI '{icon_path}' failed to parse - {err}")); - } - }; - } - { - zone!("insert_dmi"); - let dmi_arc = Arc::new(dmi); - let other_arc = dmi_arc.clone(); - // Cache it for later, saving future DMI parsing operations, which are very slow. - ICON_FILES.insert(icon_path.to_owned(), dmi_arc); - Ok(other_arc) - } + Ok(Arc::new(Icon::load(reader).map_err(|err| { + format!("DMI '{}' failed to parse - {}", icon_path, err) + })?)) + }) + .cloned() } diff --git a/src/iconforge/spritesheet.rs b/src/iconforge/spritesheet.rs index b501c3be..449192e1 100644 --- a/src/iconforge/spritesheet.rs +++ b/src/iconforge/spritesheet.rs @@ -6,7 +6,7 @@ use super::{ use crate::{ error::Error, hash::{file_hash, string_hash}, - iconforge::image_cache::cache_transformed_images, + iconforge::image_cache::{ICON_ROOT, cache_transformed_images}, }; use dashmap::{DashMap, DashSet}; use dmi::icon::{DmiVersion, Icon, IconState}; @@ -19,6 +19,7 @@ use std::{ collections::{HashMap, HashSet}, fs::File, hash::BuildHasherDefault, + path::PathBuf, sync::{Arc, Mutex, RwLock}, }; use tracy_full::zone; @@ -343,6 +344,20 @@ pub fn generate_headless(file_path: &str, sprites: &str, flatten: &str) -> Headl } } +static CREATED_DIRS: Lazy> = Lazy::new(DashSet::new); + +fn ensure_dir_exists(path: PathBuf, error: &Arc>>) { + if CREATED_DIRS.insert(path.clone()) + && let Err(err) = std::fs::create_dir_all(&path) + { + error.lock().unwrap().push(format!( + "Failed to create directory '{}': {}", + path.display(), + err + )); + } +} + pub fn generate_spritesheet( file_path: &str, spritesheet_name: &str, @@ -352,6 +367,9 @@ pub fn generate_spritesheet( flatten: &str, ) -> std::result::Result { zone!("generate_spritesheet"); + + let base_path = ICON_ROOT.join(file_path); + let hash_icons: bool = hash_icons == "1"; let generate_dmi: bool = generate_dmi == "1"; // PNGs cannot be non-flat @@ -428,14 +446,16 @@ pub fn generate_spritesheet( // cache this here so we don't generate the same string 5000 times let sprite_name = String::from("N/A, in tree generation stage"); - // Map duplicate transform sets into a tree. - // This is beneficial in the case where we have the same base image, and the same set of transforms, but change 1 or 2 things at the end. - // We can greatly reduce the amount of RgbaImages created by first finding these. - tree_bases - .lock() - .unwrap() - .par_iter() - .for_each(|(_, icons)| { + { + // Map duplicate transform sets into a tree. + // This is beneficial in the case where we have the same base image, and the same set of transforms, but change 1 or 2 things at the end. + // We can greatly reduce the amount of RgbaImages created by first finding these. + let tree_vec: Vec> = { + let guard = tree_bases.lock().unwrap(); + guard.values().cloned().collect() + }; + + tree_vec.par_iter().for_each(|icons| { zone!("transform_trees"); let first_icon = match icons.first() { Some((_, icon)) => icon, @@ -486,6 +506,7 @@ pub fn generate_spritesheet( } } }); + } // Pick the specific icon states out of the DMI, also generating their transforms, build the spritesheet metadata. sprites_map.par_iter().for_each(|sprite_entry| { @@ -543,29 +564,44 @@ pub fn generate_spritesheet( // all images have been returned now, so continue... // Get all the sprites and spew them onto a spritesheet. - size_to_icon_objects - .lock() - .unwrap() + let size_entries: Vec<(String, Vec<(&String, &UniversalIcon)>)> = { + let guard = size_to_icon_objects.lock().unwrap(); + guard.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + }; + { + zone!("precreate_dirs"); + let mut parent_dirs = std::collections::HashSet::::new(); + + for (size_id, _) in &size_entries { + let output_path = base_path.join(format!( + "{}_{}.{}", + spritesheet_name, + size_id, + if generate_dmi { "dmi" } else { "png" } + )); + if let Some(parent) = output_path.parent() { + parent_dirs.insert(parent.to_path_buf()); + } + } + + for dir in parent_dirs { + ensure_dir_exists(dir, &error); + } + } + + size_entries .par_iter() .for_each(|(size_id, sprite_entries)| { zone!("join_sprites"); - let file_path = format!( - "{file_path}{spritesheet_name}_{size_id}.{}", + let file_path = base_path.join(format!( + "{}_{}.{}", + spritesheet_name, + size_id, if generate_dmi { "dmi" } else { "png" } - ); + )); let size_data: Vec<&str> = size_id.split('x').collect(); - let base_width = size_data - .first() - .unwrap() - .to_string() - .parse::() - .unwrap(); - let base_height = size_data - .last() - .unwrap() - .to_string() - .parse::() - .unwrap(); + let base_width = size_data.first().unwrap().parse::().unwrap(); + let base_height = size_data.last().unwrap().parse::().unwrap(); if generate_dmi { let output_states = @@ -580,15 +616,14 @@ pub fn generate_spritesheet( zone!("write_spritesheet_dmi"); { zone!("create_file"); - let path = std::path::Path::new(&file_path); - if let Err(err) = std::fs::create_dir_all(path.parent().unwrap()) { - error.lock().unwrap().push(err.to_string()); - return; - }; - let mut output_file = match File::create(path) { - Ok(file) => file, + let mut output_file = match File::create(&file_path) { + Ok(f) => f, Err(err) => { - error.lock().unwrap().push(err.to_string()); + error.lock().unwrap().push(format!( + "Failed to create DMI file '{}': {}", + file_path.display(), + err + )); return; } }; @@ -616,8 +651,12 @@ pub fn generate_spritesheet( }; { zone!("write_spritesheet_png"); - if let Err(err) = final_image.save(file_path) { - error.lock().unwrap().push(err.to_string()); + if let Err(err) = final_image.save(&file_path) { + error.lock().unwrap().push(format!( + "Failed to save PNG file '{}': {}", + file_path.display(), + err + )); } } } @@ -745,7 +784,7 @@ fn transform_leaves( zone!("do_next_transforms"); next_transforms .into_par_iter() - .for_each(|(transform, mut associated_icons)| { + .for_each(|(transform, associated_icons)| { let altered_image_data = match transform.apply(image_data.clone(), flatten) { Ok(data) => Arc::new(data), Err(err) => { @@ -753,31 +792,24 @@ fn transform_leaves( return; } }; - { - zone!("filter_associated_icons"); - associated_icons - .clone() - .into_iter() - .enumerate() - .for_each(|(idx, icon)| { - if icon.transform.len() as u8 == depth + 1 - && *icon.transform.last().unwrap() == transform - { - associated_icons.swap_remove(idx); - image_cache::cache_transformed_images( - icon, - altered_image_data.clone(), - flatten, - ); - } - }); + zone!("filter_associated_icons"); + let (finished, remaining): (Vec<_>, Vec<_>) = + associated_icons.into_iter().partition(|icon| { + icon.transform.len() as u8 == depth + 1 + && *icon.transform.last().unwrap() == transform + }); + + for icon in finished { + image_cache::cache_transformed_images( + icon, + altered_image_data.clone(), + flatten, + ); } - if let Err(err) = transform_leaves( - &associated_icons, - altered_image_data.clone(), - depth + 1, - flatten, - ) { + + if let Err(err) = + transform_leaves(&remaining, altered_image_data.clone(), depth + 1, flatten) + { errors.lock().unwrap().push(err); } }); diff --git a/tests/iconforge/mod.rs b/tests/iconforge/mod.rs index ef124063..b3c7c281 100644 --- a/tests/iconforge/mod.rs +++ b/tests/iconforge/mod.rs @@ -208,7 +208,7 @@ fn compare_states(dm_state: &IconState, rustg_state: &IconState) -> Option Date: Mon, 26 Jan 2026 22:07:22 +0100 Subject: [PATCH 2/2] suggestions --- src/iconforge/spritesheet.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/iconforge/spritesheet.rs b/src/iconforge/spritesheet.rs index 449192e1..1284910f 100644 --- a/src/iconforge/spritesheet.rs +++ b/src/iconforge/spritesheet.rs @@ -599,9 +599,10 @@ pub fn generate_spritesheet( size_id, if generate_dmi { "dmi" } else { "png" } )); - let size_data: Vec<&str> = size_id.split('x').collect(); - let base_width = size_data.first().unwrap().parse::().unwrap(); - let base_height = size_data.last().unwrap().parse::().unwrap(); + let (base_width, base_height) = size_id + .split_once('x') + .map(|(w, h)| (w.parse::().unwrap(), h.parse::().unwrap())) + .unwrap(); if generate_dmi { let output_states =