zerovec/ule/
encode.rs

Help
1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::ule::*;
6use crate::varzerovec::VarZeroVecFormat;
7use crate::{VarZeroSlice, VarZeroVec, ZeroSlice, ZeroVec};
8#[cfg(feature = "alloc")]
9use alloc::borrow::{Cow, ToOwned};
10#[cfg(feature = "alloc")]
11use alloc::boxed::Box;
12#[cfg(feature = "alloc")]
13use alloc::string::String;
14#[cfg(feature = "alloc")]
15use alloc::{vec, vec::Vec};
16#[cfg(feature = "alloc")]
17use core::mem;
18
19/// Allows types to be encoded as VarULEs. This is highly useful for implementing VarULE on
20/// custom DSTs where the type cannot be obtained as a reference to some other type.
21///
22/// [`Self::encode_var_ule_as_slices()`] should be implemented by providing an encoded slice for each field
23/// of the VarULE type to the callback, in order. For an implementation to be safe, the slices
24/// to the callback must, when concatenated, be a valid instance of the VarULE type.
25///
26/// See the [custom VarULEdocumentation](crate::ule::custom) for examples.
27///
28/// [`Self::encode_var_ule_as_slices()`] is only used to provide default implementations for [`Self::encode_var_ule_write()`]
29/// and [`Self::encode_var_ule_len()`]. If you override the default implementations it is totally valid to
30/// replace [`Self::encode_var_ule_as_slices()`]'s body with `unreachable!()`. This can be done for cases where
31/// it is not possible to implement [`Self::encode_var_ule_as_slices()`] but the other methods still work.
32///
33/// A typical implementation will take each field in the order found in the [`VarULE`] type,
34/// convert it to ULE, call [`ULE::slice_as_bytes()`] on them, and pass the slices to `cb` in order.
35/// A trailing [`ZeroVec`](crate::ZeroVec) or [`VarZeroVec`](crate::VarZeroVec) can have their underlying
36/// byte representation passed through.
37///
38/// In case the compiler is not optimizing [`Self::encode_var_ule_len()`], it can be overridden. A typical
39/// implementation will add up the sizes of each field on the [`VarULE`] type and then add in the byte length of the
40/// dynamically-sized part.
41///
42/// # Reverse-encoding VarULE
43///
44/// This trait maps a struct to its bytes representation ("serialization"), and
45/// [`ZeroFrom`](zerofrom::ZeroFrom) performs the opposite operation, taking those bytes and
46/// creating a struct from them ("deserialization").
47///
48/// # Safety
49///
50/// The safety invariants of [`Self::encode_var_ule_as_slices()`] are:
51/// - It must call `cb` (only once)
52/// - The slices passed to `cb`, if concatenated, should be a valid instance of the `T` [`VarULE`] type
53///   (i.e. if fed to [`VarULE::validate_bytes()`] they must produce a successful result)
54/// - It must return the return value of `cb` to the caller
55///
56/// One or more of [`Self::encode_var_ule_len()`] and [`Self::encode_var_ule_write()`] may be provided.
57/// If both are, then `zerovec` code is guaranteed to not call [`Self::encode_var_ule_as_slices()`], and it may be replaced
58/// with `unreachable!()`.
59///
60/// The safety invariants of [`Self::encode_var_ule_len()`] are:
61/// - It must return the length of the corresponding VarULE type
62///
63/// The safety invariants of [`Self::encode_var_ule_write()`] are:
64/// - The slice written to `dst` must be a valid instance of the `T` [`VarULE`] type
65pub unsafe trait EncodeAsVarULE<T: VarULE + ?Sized> {
66    /// Calls `cb` with a piecewise list of byte slices that when concatenated
67    /// produce the memory pattern of the corresponding instance of `T`.
68    ///
69    /// Do not call this function directly; instead use the other two. Some implementors
70    /// may define this function to panic.
71    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R;
72
73    /// Return the length, in bytes, of the corresponding [`VarULE`] type
74    fn encode_var_ule_len(&self) -> usize {
75        self.encode_var_ule_as_slices(|slices| slices.iter().map(|s| s.len()).sum())
76    }
77
78    /// Write the corresponding [`VarULE`] type to the `dst` buffer. `dst` should
79    /// be the size of [`Self::encode_var_ule_len()`]
80    fn encode_var_ule_write(&self, mut dst: &mut [u8]) {
81        debug_assert_eq!(self.encode_var_ule_len(), dst.len());
82        self.encode_var_ule_as_slices(move |slices| {
83            #[allow(clippy::indexing_slicing)] // by debug_assert
84            for slice in slices {
85                dst[..slice.len()].copy_from_slice(slice);
86                dst = &mut dst[slice.len()..];
87            }
88        });
89    }
90}
91
92/// Given an [`EncodeAsVarULE`] type `S`, encode it into a `Box<T>`
93///
94/// This is primarily useful for generating `Deserialize` impls for VarULE types
95#[cfg(feature = "alloc")]
96pub fn encode_varule_to_box<S: EncodeAsVarULE<T> + ?Sized, T: VarULE + ?Sized>(x: &S) -> Box<T> {
97    // zero-fill the vector to avoid uninitialized data UB
98    let mut vec: Vec<u8> = vec![0; x.encode_var_ule_len()];
99    x.encode_var_ule_write(&mut vec);
100    let boxed = mem::ManuallyDrop::new(vec.into_boxed_slice());
101    unsafe {
102        // Safety: `ptr` is a box, and `T` is a VarULE which guarantees it has the same memory layout as `[u8]`
103        // and can be recouped via from_bytes_unchecked()
104        let ptr: *mut T = T::from_bytes_unchecked(&boxed) as *const T as *mut T;
105
106        // Safety: we can construct an owned version since we have mem::forgotten the older owner
107        Box::from_raw(ptr)
108    }
109}
110
111unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for T {
112    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
113        cb(&[T::as_bytes(self)])
114    }
115}
116
117unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for &'_ T {
118    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
119        cb(&[T::as_bytes(self)])
120    }
121}
122
123unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for &'_ &'_ T {
124    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
125        cb(&[T::as_bytes(self)])
126    }
127}
128
129#[cfg(feature = "alloc")]
130unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for Cow<'_, T>
131where
132    T: ToOwned,
133{
134    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
135        cb(&[T::as_bytes(self.as_ref())])
136    }
137}
138
139#[cfg(feature = "alloc")]
140unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for Box<T> {
141    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
142        cb(&[T::as_bytes(self)])
143    }
144}
145
146#[cfg(feature = "alloc")]
147unsafe impl<T: VarULE + ?Sized> EncodeAsVarULE<T> for &'_ Box<T> {
148    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
149        cb(&[T::as_bytes(self)])
150    }
151}
152
153#[cfg(feature = "alloc")]
154unsafe impl EncodeAsVarULE<str> for String {
155    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
156        cb(&[self.as_bytes()])
157    }
158}
159
160#[cfg(feature = "alloc")]
161unsafe impl EncodeAsVarULE<str> for &'_ String {
162    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
163        cb(&[self.as_bytes()])
164    }
165}
166
167// Note: This impl could technically use `T: AsULE`, but we want users to prefer `ZeroSlice<T>`
168// for cases where T is not a ULE. Therefore, we can use the more efficient `memcpy` impl here.
169#[cfg(feature = "alloc")]
170unsafe impl<T> EncodeAsVarULE<[T]> for Vec<T>
171where
172    T: ULE,
173{
174    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
175        cb(&[<[T] as VarULE>::as_bytes(self)])
176    }
177}
178
179unsafe impl<T> EncodeAsVarULE<ZeroSlice<T>> for &'_ [T]
180where
181    T: AsULE + 'static,
182{
183    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
184        // unnecessary if the other two are implemented
185        unreachable!()
186    }
187
188    #[inline]
189    fn encode_var_ule_len(&self) -> usize {
190        self.len() * core::mem::size_of::<T::ULE>()
191    }
192
193    fn encode_var_ule_write(&self, dst: &mut [u8]) {
194        #[allow(non_snake_case)]
195        let S = core::mem::size_of::<T::ULE>();
196        debug_assert_eq!(self.len() * S, dst.len());
197        for (item, ref mut chunk) in self.iter().zip(dst.chunks_mut(S)) {
198            let ule = item.to_unaligned();
199            chunk.copy_from_slice(ULE::slice_as_bytes(core::slice::from_ref(&ule)));
200        }
201    }
202}
203
204#[cfg(feature = "alloc")]
205unsafe impl<T> EncodeAsVarULE<ZeroSlice<T>> for Vec<T>
206where
207    T: AsULE + 'static,
208{
209    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
210        // unnecessary if the other two are implemented
211        unreachable!()
212    }
213
214    #[inline]
215    fn encode_var_ule_len(&self) -> usize {
216        self.as_slice().encode_var_ule_len()
217    }
218
219    #[inline]
220    fn encode_var_ule_write(&self, dst: &mut [u8]) {
221        self.as_slice().encode_var_ule_write(dst)
222    }
223}
224
225unsafe impl<T> EncodeAsVarULE<ZeroSlice<T>> for ZeroVec<'_, T>
226where
227    T: AsULE + 'static,
228{
229    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
230        // unnecessary if the other two are implemented
231        unreachable!()
232    }
233
234    #[inline]
235    fn encode_var_ule_len(&self) -> usize {
236        self.as_bytes().len()
237    }
238
239    fn encode_var_ule_write(&self, dst: &mut [u8]) {
240        debug_assert_eq!(self.as_bytes().len(), dst.len());
241        dst.copy_from_slice(self.as_bytes());
242    }
243}
244
245unsafe impl<T, E, F> EncodeAsVarULE<VarZeroSlice<T, F>> for &'_ [E]
246where
247    T: VarULE + ?Sized,
248    E: EncodeAsVarULE<T>,
249    F: VarZeroVecFormat,
250{
251    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
252        // unnecessary if the other two are implemented
253        unimplemented!()
254    }
255
256    #[allow(clippy::unwrap_used)] // TODO(#1410): Rethink length errors in VZV.
257    fn encode_var_ule_len(&self) -> usize {
258        crate::varzerovec::components::compute_serializable_len::<T, E, F>(self).unwrap() as usize
259    }
260
261    fn encode_var_ule_write(&self, dst: &mut [u8]) {
262        crate::varzerovec::components::write_serializable_bytes::<T, E, F>(self, dst)
263    }
264}
265
266#[cfg(feature = "alloc")]
267unsafe impl<T, E, F> EncodeAsVarULE<VarZeroSlice<T, F>> for Vec<E>
268where
269    T: VarULE + ?Sized,
270    E: EncodeAsVarULE<T>,
271    F: VarZeroVecFormat,
272{
273    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
274        // unnecessary if the other two are implemented
275        unreachable!()
276    }
277
278    #[inline]
279    fn encode_var_ule_len(&self) -> usize {
280        <_ as EncodeAsVarULE<VarZeroSlice<T, F>>>::encode_var_ule_len(&self.as_slice())
281    }
282
283    #[inline]
284    fn encode_var_ule_write(&self, dst: &mut [u8]) {
285        <_ as EncodeAsVarULE<VarZeroSlice<T, F>>>::encode_var_ule_write(&self.as_slice(), dst)
286    }
287}
288
289unsafe impl<T, F> EncodeAsVarULE<VarZeroSlice<T, F>> for VarZeroVec<'_, T, F>
290where
291    T: VarULE + ?Sized,
292    F: VarZeroVecFormat,
293{
294    fn encode_var_ule_as_slices<R>(&self, _: impl FnOnce(&[&[u8]]) -> R) -> R {
295        // unnecessary if the other two are implemented
296        unreachable!()
297    }
298
299    #[inline]
300    fn encode_var_ule_len(&self) -> usize {
301        self.as_bytes().len()
302    }
303
304    #[inline]
305    fn encode_var_ule_write(&self, dst: &mut [u8]) {
306        debug_assert_eq!(self.as_bytes().len(), dst.len());
307        dst.copy_from_slice(self.as_bytes());
308    }
309}
310
311#[cfg(test)]
312mod test {
313    use super::*;
314
315    const STRING_ARRAY: [&str; 2] = ["hello", "world"];
316
317    const STRING_SLICE: &[&str] = &STRING_ARRAY;
318
319    const U8_ARRAY: [u8; 8] = [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07];
320
321    const U8_2D_ARRAY: [&[u8]; 2] = [&U8_ARRAY, &U8_ARRAY];
322
323    const U8_2D_SLICE: &[&[u8]] = &[&U8_ARRAY, &U8_ARRAY];
324
325    const U8_3D_ARRAY: [&[&[u8]]; 2] = [U8_2D_SLICE, U8_2D_SLICE];
326
327    const U8_3D_SLICE: &[&[&[u8]]] = &[U8_2D_SLICE, U8_2D_SLICE];
328
329    const U32_ARRAY: [u32; 4] = [0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F];
330
331    const U32_2D_ARRAY: [&[u32]; 2] = [&U32_ARRAY, &U32_ARRAY];
332
333    const U32_2D_SLICE: &[&[u32]] = &[&U32_ARRAY, &U32_ARRAY];
334
335    const U32_3D_ARRAY: [&[&[u32]]; 2] = [U32_2D_SLICE, U32_2D_SLICE];
336
337    const U32_3D_SLICE: &[&[&[u32]]] = &[U32_2D_SLICE, U32_2D_SLICE];
338
339    #[test]
340    fn test_vzv_from() {
341        type VZV<'a, T> = VarZeroVec<'a, T>;
342        type ZS<T> = ZeroSlice<T>;
343        type VZS<T> = VarZeroSlice<T>;
344
345        let u8_zerovec: ZeroVec<u8> = ZeroVec::from_slice_or_alloc(&U8_ARRAY);
346        let u8_2d_zerovec: [ZeroVec<u8>; 2] = [u8_zerovec.clone(), u8_zerovec.clone()];
347        let u8_2d_vec: Vec<Vec<u8>> = vec![U8_ARRAY.into(), U8_ARRAY.into()];
348        let u8_3d_vec: Vec<Vec<Vec<u8>>> = vec![u8_2d_vec.clone(), u8_2d_vec.clone()];
349
350        let u32_zerovec: ZeroVec<u32> = ZeroVec::from_slice_or_alloc(&U32_ARRAY);
351        let u32_2d_zerovec: [ZeroVec<u32>; 2] = [u32_zerovec.clone(), u32_zerovec.clone()];
352        let u32_2d_vec: Vec<Vec<u32>> = vec![U32_ARRAY.into(), U32_ARRAY.into()];
353        let u32_3d_vec: Vec<Vec<Vec<u32>>> = vec![u32_2d_vec.clone(), u32_2d_vec.clone()];
354
355        let a: VZV<str> = VarZeroVec::from(&STRING_ARRAY);
356        let b: VZV<str> = VarZeroVec::from(STRING_SLICE);
357        let c: VZV<str> = VarZeroVec::from(&Vec::from(STRING_SLICE));
358        assert_eq!(a, STRING_SLICE);
359        assert_eq!(a, b);
360        assert_eq!(a, c);
361
362        let a: VZV<[u8]> = VarZeroVec::from(&U8_2D_ARRAY);
363        let b: VZV<[u8]> = VarZeroVec::from(U8_2D_SLICE);
364        let c: VZV<[u8]> = VarZeroVec::from(&u8_2d_vec);
365        assert_eq!(a, U8_2D_SLICE);
366        assert_eq!(a, b);
367        assert_eq!(a, c);
368        let u8_3d_vzv_brackets = &[a.clone(), a.clone()];
369
370        let a: VZV<ZS<u8>> = VarZeroVec::from(&U8_2D_ARRAY);
371        let b: VZV<ZS<u8>> = VarZeroVec::from(U8_2D_SLICE);
372        let c: VZV<ZS<u8>> = VarZeroVec::from(&u8_2d_vec);
373        let d: VZV<ZS<u8>> = VarZeroVec::from(&u8_2d_zerovec);
374        assert_eq!(a, U8_2D_SLICE);
375        assert_eq!(a, b);
376        assert_eq!(a, c);
377        assert_eq!(a, d);
378        let u8_3d_vzv_zeroslice = &[a.clone(), a.clone()];
379
380        let a: VZV<VZS<[u8]>> = VarZeroVec::from(&U8_3D_ARRAY);
381        let b: VZV<VZS<[u8]>> = VarZeroVec::from(U8_3D_SLICE);
382        let c: VZV<VZS<[u8]>> = VarZeroVec::from(&u8_3d_vec);
383        let d: VZV<VZS<[u8]>> = VarZeroVec::from(u8_3d_vzv_brackets);
384        assert_eq!(
385            a.iter()
386                .map(|x| x.iter().map(|y| y.to_vec()).collect::<Vec<Vec<u8>>>())
387                .collect::<Vec<Vec<Vec<u8>>>>(),
388            u8_3d_vec
389        );
390        assert_eq!(a, b);
391        assert_eq!(a, c);
392        assert_eq!(a, d);
393
394        let a: VZV<VZS<ZS<u8>>> = VarZeroVec::from(&U8_3D_ARRAY);
395        let b: VZV<VZS<ZS<u8>>> = VarZeroVec::from(U8_3D_SLICE);
396        let c: VZV<VZS<ZS<u8>>> = VarZeroVec::from(&u8_3d_vec);
397        let d: VZV<VZS<ZS<u8>>> = VarZeroVec::from(u8_3d_vzv_zeroslice);
398        assert_eq!(
399            a.iter()
400                .map(|x| x
401                    .iter()
402                    .map(|y| y.iter().collect::<Vec<u8>>())
403                    .collect::<Vec<Vec<u8>>>())
404                .collect::<Vec<Vec<Vec<u8>>>>(),
405            u8_3d_vec
406        );
407        assert_eq!(a, b);
408        assert_eq!(a, c);
409        assert_eq!(a, d);
410
411        let a: VZV<ZS<u32>> = VarZeroVec::from(&U32_2D_ARRAY);
412        let b: VZV<ZS<u32>> = VarZeroVec::from(U32_2D_SLICE);
413        let c: VZV<ZS<u32>> = VarZeroVec::from(&u32_2d_vec);
414        let d: VZV<ZS<u32>> = VarZeroVec::from(&u32_2d_zerovec);
415        assert_eq!(a, u32_2d_zerovec);
416        assert_eq!(a, b);
417        assert_eq!(a, c);
418        assert_eq!(a, d);
419        let u32_3d_vzv = &[a.clone(), a.clone()];
420
421        let a: VZV<VZS<ZS<u32>>> = VarZeroVec::from(&U32_3D_ARRAY);
422        let b: VZV<VZS<ZS<u32>>> = VarZeroVec::from(U32_3D_SLICE);
423        let c: VZV<VZS<ZS<u32>>> = VarZeroVec::from(&u32_3d_vec);
424        let d: VZV<VZS<ZS<u32>>> = VarZeroVec::from(u32_3d_vzv);
425        assert_eq!(
426            a.iter()
427                .map(|x| x
428                    .iter()
429                    .map(|y| y.iter().collect::<Vec<u32>>())
430                    .collect::<Vec<Vec<u32>>>())
431                .collect::<Vec<Vec<Vec<u32>>>>(),
432            u32_3d_vec
433        );
434        assert_eq!(a, b);
435        assert_eq!(a, c);
436        assert_eq!(a, d);
437    }
438}
zerovec/ule/encode.rs

zerovec/ule/
encode.rs
