unbanished_formats/
pkg.rs

1//! Packages (`.pkg`, and mods as `.pkm`): the archives that hold the game's data.
2//!
3//! Original: `IO::PackageFile::ReadPackfile`, Runtime `0x1800b6f10`.
4//!
5//! ```text
6//! u32 header        bit31 = extended header, bits0..30 = TOC offset
7//! ...file data      each entry a block stream, except `.index` stubs and raw `.str` audio
8//! u32 tocSize       at TOC offset, followed by the TOC as a block stream
9//! ```
10//! Decoded TOC: `u32 count, u32 version, u32 gameVersion, u32 userVersion, u32 priority,
11//! u32 skip, u8 canOverride, u32 crc, u32 iconW, u32 iconH, icon RGBA,
12//! [v>1] u32 previewLen + bytes, [v>2] name/author/description (u32 len + bytes),
13//! count x { u32 nameLen, name, u32 offset, u32 size }`.
14
15use crate::error::{format_err, Error, Result};
16use crate::stream;
17
18/// One entry of a package: its name, and where its bytes are.
19#[derive(Debug, Clone)]
20pub struct Entry {
21    /// Original path with backslashes, e.g. `Models\Animals\Sheep\SheepMesh.rsc:resource`... as stored.
22    pub name: String,
23    /// Where the entry's bytes start in the package.
24    pub offset: usize,
25    /// The size of the stored bytes.
26    pub size: usize,
27}
28
29impl Entry {
30    /// How the entry's bytes are stored.
31    pub fn kind(&self) -> EntryKind {
32        if self.name.ends_with(".index") {
33            EntryKind::Index
34        } else if self.name.ends_with(".str") || self.name.ends_with(".sav") {
35            // Audio streams and the tutorials' saves (`Save\TutorialFood.sav`), stored as is.
36            EntryKind::RawStream
37        } else {
38            EntryKind::Compressed
39        }
40    }
41
42    /// Normalised resource key: forward slashes, lower case, `:` kept.
43    pub fn key(&self) -> String {
44        normalise(&self.name)
45    }
46}
47
48/// How an entry is stored.
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub enum EntryKind {
51    /// 64-byte build-path stub; not a resource.
52    Index,
53    /// Stored uncompressed (streamed music, xWMA).
54    RawStream,
55    /// Block stream (`.crs`, `.sav`).
56    Compressed,
57}
58
59/// The header of a package: versions, priority, and a mod's name, icon and description.
60#[derive(Debug, Clone)]
61pub struct Header {
62    /// The package format version.
63    pub version: u32,
64    // +0x68; +0x6c
65    /// The game version the package was made with (`0x010007ff` is 1.0.7; read with
66    /// `Configuration::GetVersionString`), the author's version number (a mod's).
67    pub game_version: u32,
68    /// The author's version number (a mod's).
69    pub user_version: u32,
70    /// The priority: the entry of the package with the higher priority wins (10,000 for mods).
71    pub priority: u32,
72    // +0x74
73    /// `Modification::RefreshList` leaves out a mod whose value is not 0 (0 in every package seen;
74    /// its meaning is not traced).
75    pub skip: u32,
76    // +0x78/+0x7c/+0x80
77    /// The icon (48×48 for mods), `0xAARRGGBB` texels.
78    pub icon: (u32, u32, Vec<u8>),
79    // +0x98
80    /// The preview image's file (jpg or png), from version 2.
81    pub preview: Vec<u8>,
82    /// Whether a mod may replace this package's entries.
83    pub can_override: bool,
84    /// A checksum of the package.
85    pub crc: u32,
86    /// A mod's name.
87    pub name: String,
88    /// A mod's author.
89    pub author: String,
90    /// A mod's description.
91    pub description: String,
92}
93
94/// One package: its header, its entries and its bytes.
95pub struct Package {
96    data: Vec<u8>,
97    /// The header.
98    pub header: Header,
99    /// The entries, in the order of the table of contents.
100    pub entries: Vec<Entry>,
101}
102
103/// The normalised form of an entry name or a reference: forward slashes, lower case.
104pub fn normalise(name: &str) -> String {
105    name.replace('\\', "/").to_ascii_lowercase()
106}
107
108struct Cursor<'a> {
109    d: &'a [u8],
110    p: usize,
111}
112
113impl<'a> Cursor<'a> {
114    fn take(&mut self, n: usize) -> Result<&'a [u8]> {
115        let s = self.d.get(self.p..self.p + n).ok_or(Error::Truncated {
116            at: self.p,
117            wanted: n,
118        })?;
119        self.p += n;
120        Ok(s)
121    }
122    fn u32(&mut self) -> Result<u32> {
123        Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap()))
124    }
125    fn cstr(&mut self, n: usize) -> Result<String> {
126        let b = self.take(n)?;
127        let end = b.iter().position(|&c| c == 0).unwrap_or(b.len());
128        Ok(String::from_utf8(b[..end].to_vec())
129            .unwrap_or_else(|_| b[..end].iter().map(|&c| c as char).collect()))
130    }
131    /// `n` bytes of UTF-16 (packages before version 3), to the first NUL.
132    fn wstr(&mut self, n: usize) -> Result<String> {
133        let b = self.take(n)?;
134        let units: Vec<u16> = b
135            .chunks_exact(2)
136            .map(|c| u16::from_le_bytes([c[0], c[1]]))
137            .take_while(|&u| u != 0)
138            .collect();
139        Ok(String::from_utf16_lossy(&units))
140    }
141    /// A length-prefixed string: UTF-8 bytes from version 3, UTF-16 before.
142    fn string(&mut self, version: u32) -> Result<String> {
143        let n = self.u32()? as usize;
144        if version < 3 {
145            self.wstr(n)
146        } else {
147            self.cstr(n)
148        }
149    }
150}
151
152impl Package {
153    /// Read a package from its bytes.
154    pub fn parse(data: Vec<u8>) -> Result<Package> {
155        let head = Cursor { d: &data, p: 0 }.u32()?;
156        if head >> 31 == 0 {
157            return Err(Error::Unsupported("legacy package header".into()));
158        }
159        let toc_at = (head & 0x7FFF_FFFF) as usize;
160        let toc_size = Cursor {
161            d: &data,
162            p: toc_at,
163        }
164        .u32()? as usize;
165        let toc_bytes = data
166            .get(toc_at + 4..toc_at + 4 + toc_size)
167            .ok_or(Error::Truncated {
168                at: toc_at + 4,
169                wanted: toc_size,
170            })?;
171        let toc = stream::decode(toc_bytes)?;
172        let mut c = Cursor { d: &toc, p: 0 };
173
174        // `PackageFile::ReadPackfile` (Runtime `0x1800b6f10`): the preview from version 2,
175        // strings and names as UTF-8 from version 3 (UTF-16 before). Mods (`.pkm`) of every
176        // version use this layout.
177        let count = c.u32()? as usize;
178        let version = c.u32()?;
179        if version == 0 {
180            return Err(Error::Unsupported(format!("package version {version}")));
181        }
182        let game_version = c.u32()?;
183        let user_version = c.u32()?;
184        let priority = c.u32()?;
185        let skip = c.u32()?;
186        let can_override = c.take(1)?[0] != 0;
187        let crc = c.u32()?;
188        let (w, h) = (c.u32()? as usize, c.u32()? as usize);
189        let icon = c.take(w * h * 4)?.to_vec();
190        let preview = if version > 1 {
191            let n = c.u32()? as usize;
192            // Some third-party packers leave the length unset (`0xe1e1e1e1`) with no image
193            // after it: inferred, a length past the table is no preview.
194            if n > c.d.len() - c.p {
195                Vec::new()
196            } else {
197                c.take(n)?.to_vec()
198            }
199        } else {
200            Vec::new()
201        };
202        let mut strings = [String::new(), String::new(), String::new()];
203        for s in &mut strings {
204            *s = c.string(version)?;
205        }
206        let [name, author, description] = strings;
207
208        let mut entries = Vec::with_capacity(count);
209        for _ in 0..count {
210            let name = c.string(version)?;
211            let offset = c.u32()? as usize;
212            let size = c.u32()? as usize;
213            if offset + size > data.len() {
214                return format_err(format!("entry {name} extends past end of package"));
215            }
216            entries.push(Entry { name, offset, size });
217        }
218        if c.p != toc.len() {
219            return format_err(format!("{} unparsed TOC bytes", toc.len() - c.p));
220        }
221        let header = Header {
222            version,
223            game_version,
224            user_version,
225            priority,
226            skip,
227            icon: (w as u32, h as u32, icon),
228            preview,
229            can_override,
230            crc,
231            name,
232            author,
233            description,
234        };
235        Ok(Package {
236            data,
237            header,
238            entries,
239        })
240    }
241
242    /// An entry's bytes as stored (still compressed for most entries).
243    pub fn raw(&self, entry: &Entry) -> &[u8] {
244        &self.data[entry.offset..entry.offset + entry.size]
245    }
246
247    /// Entry contents: decompressed for block streams, as stored otherwise.
248    pub fn read(&self, entry: &Entry) -> Result<Vec<u8>> {
249        match entry.kind() {
250            EntryKind::Compressed => stream::decode(self.raw(entry)),
251            EntryKind::RawStream | EntryKind::Index => Ok(self.raw(entry).to_vec()),
252        }
253    }
254}
255
256/// A set of packages resolved by priority: higher priority shadows lower for the same name.
257#[derive(Default)]
258pub struct Archive {
259    packages: Vec<Package>,
260    index: std::collections::HashMap<String, (usize, usize)>,
261    // IO::FileSystem +0x38
262    /// `IO::FileSystem`: the user pack files (enabled mods) the packages were set up with
263    /// (`AddUserPackFile`, `AreUserPackFilesEnabled`).
264    pub user_packs: usize,
265    /// How many of the packages are mods (added last, with [`Archive::add_mod`]).
266    mods: usize,
267}
268
269impl Archive {
270    /// Add one of the game's packages. An entry replaces one of the same name when its package's
271    /// priority is not lower.
272    pub fn add(&mut self, pkg: Package) {
273        let pi = self.packages.len();
274        for (ei, e) in pkg.entries.iter().enumerate() {
275            if e.kind() == EntryKind::Index {
276                continue;
277            }
278            let key = e.key();
279            let replace = match self.index.get(&key) {
280                Some(&(p, _)) => pkg.header.priority >= self.packages[p].header.priority,
281                None => true,
282            };
283            if replace {
284                self.index.insert(key, (pi, ei));
285            }
286        }
287        self.packages.push(pkg);
288        self.mods += 1;
289    }
290
291    // +0xac
292    /// `FileSystem::LoadPackages` for a mod (`.pkm`), added after the game's packages: the enabled
293    /// mods go in from the last in the mod list to the first, and a file replaces one already there
294    /// only when that one's package allows it. So the first mod in the list wins, and nothing
295    /// overrides `data1` or `patch1`.
296    ///
297    /// Original: Runtime `0x1800ac3b0`.
298    pub fn add_mod(&mut self, pkg: Package) {
299        let pi = self.packages.len();
300        for (ei, e) in pkg.entries.iter().enumerate() {
301            if e.kind() == EntryKind::Index {
302                continue;
303            }
304            let key = e.key();
305            let replace = match self.index.get(&key) {
306                Some(&(p, _)) => self.packages[p].header.can_override,
307                None => true,
308            };
309            if replace {
310                self.index.insert(key, (pi, ei));
311            }
312        }
313        self.packages.push(pkg);
314    }
315
316    /// Fingerprints of the game's packages and of the mods (not in the original): the CRC-32 of
317    /// every entry's name and size, package by package in load order. Two games with the same
318    /// data and the same mods in the same order have the same fingerprints; multiplayer compares
319    /// them before players share a game.
320    pub fn fingerprints(&self) -> (u32, u32) {
321        let game = self.packages.len() - self.mods;
322        let print = |packages: &[Package]| {
323            let mut c = !0u32;
324            for p in packages {
325                for e in &p.entries {
326                    c = crate::crc::crc32_raw(c, e.name.as_bytes());
327                    c = crate::crc::crc32_raw(c, &(e.size as u64).to_le_bytes());
328                }
329            }
330            !c
331        };
332        (print(&self.packages[..game]), print(&self.packages[game..]))
333    }
334
335    /// How many packages are loaded (the game's first, then the mods).
336    pub fn package_count(&self) -> usize {
337        self.packages.len()
338    }
339
340    /// Drop every package after the first `keep` (the mods), as `FileSystem::ClosePackages`
341    /// before the packages load again.
342    pub fn remove_after(&mut self, keep: usize) {
343        let packages: Vec<Package> = std::mem::take(&mut self.packages);
344        self.index.clear();
345        self.mods = 0;
346        for p in packages.into_iter().take(keep) {
347            self.add(p);
348        }
349    }
350
351    /// The package holding `name`'s current entry (its header names the mod).
352    pub fn package_of(&self, name: &str) -> Option<&Package> {
353        self.index
354            .get(&normalise(name))
355            .map(|&(p, _)| &self.packages[p])
356    }
357
358    /// The number of entries.
359    pub fn len(&self) -> usize {
360        self.index.len()
361    }
362
363    /// Whether there are no entries.
364    pub fn is_empty(&self) -> bool {
365        self.index.is_empty()
366    }
367
368    /// Resource names, sorted.
369    pub fn names(&self) -> Vec<&str> {
370        let mut v: Vec<&str> = self
371            .index
372            .values()
373            .map(|&(p, e)| self.packages[p].entries[e].name.as_str())
374            .collect();
375        v.sort_unstable();
376        v
377    }
378
379    /// An entry by name.
380    pub fn get(&self, name: &str) -> Option<&Entry> {
381        self.index
382            .get(&normalise(name))
383            .map(|&(p, e)| &self.packages[p].entries[e])
384    }
385
386    /// An entry's bytes, expanded, by entry name (for example `Dialog\Toolbar_resource.crs`).
387    pub fn read(&self, name: &str) -> Result<Vec<u8>> {
388        let &(p, e) = self
389            .index
390            .get(&normalise(name))
391            .ok_or_else(|| Error::Format(format!("no such resource: {name}")))?;
392        self.packages[p].read(&self.packages[p].entries[e])
393    }
394
395    /// Read the compiled resource behind a reference such as `Models\X\Y.rsc:resource`.
396    /// The package stores it as `Models\X\Y_resource.crs`.
397    pub fn read_ref(&self, reference: &str) -> Result<Vec<u8>> {
398        self.read(&ref_to_entry(reference))
399    }
400}
401
402/// `Dir\Name.rsc:object` -> `Dir\Name_object.crs`.
403pub fn ref_to_entry(reference: &str) -> String {
404    match reference.rsplit_once(':') {
405        Some((path, object)) => {
406            let stem = path.strip_suffix(".rsc").unwrap_or(path);
407            format!("{stem}_{object}.crs")
408        }
409        None => reference.to_string(),
410    }
411}
412
413#[cfg(test)]
414mod tests {
415    use super::*;
416
417    #[test]
418    fn reference_names() {
419        assert_eq!(
420            ref_to_entry(r"Models\Animals\Sheep\MaterialInstance\Sheep.rsc:resource"),
421            r"Models\Animals\Sheep\MaterialInstance\Sheep_resource.crs"
422        );
423    }
424}
unbanished_formats/pkg.rs

unbanished_formats/
pkg.rs
