fuchsia_pkg/
package_build_manifest.rs

1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::errors::PackageBuildManifestError;
6use serde::{Deserialize, Serialize};
7use std::collections::{BTreeMap, HashSet, btree_map};
8use std::fs;
9use std::io::{self, Read};
10use std::path::Path;
11use walkdir::WalkDir;
12
13/// Package file list
14///
15/// A `PackageBuildManifest` lists the files that should be included in a Fuchsia package. Both
16/// `external_contents` and `far_contents` are maps from package resource paths in the to-be-created
17/// package to paths on the local filesystem. Package resource paths start with "meta/" if and only
18/// if they are in `far_contents`.
19#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
20#[serde(transparent)]
21pub struct PackageBuildManifest(VersionedPackageBuildManifest);
22
23impl PackageBuildManifest {
24    /// Creates a `PackageBuildManifest` from external and far contents maps.
25    ///
26    /// `external_contents` is a map from package resource paths to their locations
27    /// on the host filesystem. These are the files that will be listed in
28    /// `meta/contents`. Resource paths in `external_contents` must *not* start with
29    /// `meta/` or be exactly `meta`. Resource paths must not have file/directory collisions,
30    /// e.g. ["foo", "foo/bar"] have a file/directory collision at "foo", or they will be rejected.
31    /// `far_contents` is a map from package resource paths to their locations
32    /// on the host filesystem. These are the files that will be included bodily in the
33    /// package `meta.far` archive. Resource paths in `far_contents` must start with
34    /// `meta/`.
35    ///
36    /// # Examples
37    ///
38    /// ```
39    /// # use fuchsia_pkg::PackageBuildManifest;
40    /// # use std::collections::BTreeMap;
41    /// let external_contents = BTreeMap::from([
42    ///     ("lib/mylib.so".to_string(), "build/system/path/mylib.so".to_string()),
43    /// ]);
44    /// let far_contents = BTreeMap::from([
45    ///     ("meta/my_component_manifest.cm".to_string(),
46    ///         "other/build/system/path/my_component_manifest.cm".to_string()),
47    /// ]);
48    /// let creation_manifest =
49    ///     PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
50    ///         .unwrap();
51    /// ```
52    pub fn from_external_and_far_contents(
53        external_contents: BTreeMap<String, String>,
54        far_contents: BTreeMap<String, String>,
55    ) -> Result<Self, PackageBuildManifestError> {
56        for (resource_path, _) in external_contents.iter().chain(far_contents.iter()) {
57            fuchsia_url::Resource::validate_str(resource_path).map_err(|e| {
58                PackageBuildManifestError::ResourcePath {
59                    cause: e,
60                    path: resource_path.to_string(),
61                }
62            })?;
63        }
64        let external_paths =
65            external_contents.keys().map(|path| path.as_str()).collect::<HashSet<_>>();
66        for resource_path in &external_paths {
67            if resource_path.starts_with("meta/") || resource_path.eq(&"meta") {
68                return Err(PackageBuildManifestError::ExternalContentInMetaDirectory {
69                    path: resource_path.to_string(),
70                });
71            }
72            for (i, _) in resource_path.match_indices('/') {
73                if external_paths.contains(&resource_path[..i]) {
74                    return Err(PackageBuildManifestError::FileDirectoryCollision {
75                        path: resource_path[..i].to_string(),
76                    });
77                }
78            }
79        }
80        for (resource_path, _) in far_contents.iter() {
81            if !resource_path.starts_with("meta/") {
82                return Err(PackageBuildManifestError::FarContentNotInMetaDirectory {
83                    path: resource_path.to_string(),
84                });
85            }
86        }
87        Ok(PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
88            external_contents,
89            far_contents,
90        })))
91    }
92
93    /// Deserializes a `PackageBuildManifest` from versioned json.
94    ///
95    /// # Examples
96    /// ```
97    /// # use fuchsia_pkg::PackageBuildManifest;
98    /// let json_string = r#"
99    /// {
100    ///   "version": "1",
101    ///   "content": {
102    ///     "/": {
103    ///       "lib/mylib.so": "build/system/path/mylib.so"
104    ///     },
105    ///    "/meta/": {
106    ///      "my_component_manifest.cml": "other/build/system/path/my_component_manifest.cml"
107    ///    }
108    /// }"#;
109    /// let creation_manifest = PackageBuildManifest::from_json(json_string.as_bytes());
110    /// ```
111    pub fn from_json<R: io::Read>(reader: R) -> Result<Self, PackageBuildManifestError> {
112        match serde_json::from_reader::<R, VersionedPackageBuildManifest>(reader)? {
113            VersionedPackageBuildManifest::Version1(v1) => PackageBuildManifest::from_v1(v1),
114        }
115    }
116
117    fn from_v1(v1: PackageBuildManifestV1) -> Result<Self, PackageBuildManifestError> {
118        let mut far_contents = BTreeMap::new();
119        // Validate package resource paths in far contents before "meta/" is prepended
120        // for better error messages.
121        for (resource_path, host_path) in v1.far_contents.into_iter() {
122            fuchsia_url::Resource::validate_str(&resource_path).map_err(|e| {
123                PackageBuildManifestError::ResourcePath {
124                    cause: e,
125                    path: resource_path.to_string(),
126                }
127            })?;
128            far_contents.insert(format!("meta/{resource_path}"), host_path);
129        }
130        PackageBuildManifest::from_external_and_far_contents(v1.external_contents, far_contents)
131    }
132
133    pub fn from_dir(root: impl AsRef<Path>) -> Result<Self, PackageBuildManifestError> {
134        let root = root.as_ref();
135        let mut far_contents = BTreeMap::new();
136        let mut external_contents = BTreeMap::new();
137
138        for entry in WalkDir::new(root) {
139            let entry = entry?;
140            let path = entry.path();
141            let file_type = entry.file_type();
142            if file_type.is_dir() {
143                continue;
144            }
145            if !(file_type.is_file() || file_type.is_symlink()) {
146                return Err(PackageBuildManifestError::InvalidFileType {
147                    path: path.to_path_buf(),
148                });
149            }
150
151            let relative_path = path
152                .strip_prefix(root)?
153                .to_str()
154                .ok_or(PackageBuildManifestError::EmptyResourcePath)?;
155            let path =
156                path.to_str().ok_or(PackageBuildManifestError::EmptyResourcePath)?.to_owned();
157            if relative_path.starts_with("meta") {
158                far_contents.insert(relative_path.to_owned(), path);
159            } else {
160                external_contents.insert(relative_path.to_owned(), path);
161            }
162        }
163
164        PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
165    }
166
167    /// Create a `PackageBuildManifest` from a `pm-build`-style Fuchsia INI file (fini). fini is a
168    /// simple format where each line is an entry of `$PKG_PATH=$HOST_PATH`. This copies the
169    /// parsing algorithm from pm, where:
170    ///
171    /// * The $PKG_PATH is the string up to the first equals sign.
172    /// * If there is a duplicate entry, check if the two entries have the same file contents. If
173    ///   not, error out.
174    /// * Only check if the files exist upon duplicate entry.
175    /// * Ignores lines without an '=' in it.
176    ///
177    /// Note: This functionality exists only to ease the migration from `pm build`. This will be
178    /// removed once there are no more users of `pm build`-style manifests.
179    ///
180    /// # Examples
181    ///
182    /// ```
183    /// # use fuchsia_pkg::PackageBuildManifest;
184    /// let fini_string = "\
185    ///     lib/mylib.so=build/system/path/mylib.so\n\
186    ///     meta/my_component_manifest.cml=other/build/system/path/my_component_manifest.cml\n";
187    ///
188    /// let creation_manifest = PackageBuildManifest::from_pm_fini(fini_string.as_bytes()).unwrap();
189    /// ```
190    pub fn from_pm_fini<R: io::BufRead>(mut reader: R) -> Result<Self, PackageBuildManifestError> {
191        let mut external_contents = BTreeMap::new();
192        let mut far_contents = BTreeMap::new();
193
194        let mut buf = String::new();
195        while reader.read_line(&mut buf)? != 0 {
196            let line = buf.trim();
197            if line.is_empty() {
198                buf.clear();
199                continue;
200            }
201
202            // pm's build manifest finds the first '='. If one doesn't exist the line is ignored.
203            let pos = if let Some(pos) = line.find('=') {
204                pos
205            } else {
206                buf.clear();
207                continue;
208            };
209
210            let package_path = line[..pos].trim().to_string();
211            let host_path = line[pos + 1..].trim().to_string();
212
213            let entry = if package_path.starts_with("meta/") {
214                far_contents.entry(package_path)
215            } else {
216                external_contents.entry(package_path)
217            };
218
219            match entry {
220                btree_map::Entry::Vacant(entry) => {
221                    entry.insert(host_path);
222                }
223                btree_map::Entry::Occupied(entry) => {
224                    // `pm build` manifests allow for duplicate entries, as long as they point to
225                    // the same file.
226                    if !same_file_contents(Path::new(&entry.get()), Path::new(&host_path))? {
227                        return Err(PackageBuildManifestError::DuplicateResourcePath {
228                            path: entry.key().clone(),
229                        });
230                    }
231                }
232            }
233
234            buf.clear();
235        }
236
237        Self::from_external_and_far_contents(external_contents, far_contents)
238    }
239
240    /// `external_contents` lists the blobs that make up the meta/contents file
241    pub fn external_contents(&self) -> &BTreeMap<String, String> {
242        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
243        &manifest.external_contents
244    }
245
246    /// `far_contents` lists the files to be included bodily in the meta.far
247    pub fn far_contents(&self) -> &BTreeMap<String, String> {
248        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
249        &manifest.far_contents
250    }
251}
252
253// It's possible for the same host file to be discovered through multiple paths. This is allowed as
254// long as the files have the same file contents.
255fn same_file_contents(lhs: &Path, rhs: &Path) -> io::Result<bool> {
256    // First, check if the two paths are the same.
257    if lhs == rhs {
258        return Ok(true);
259    }
260
261    // Next, check if the paths point at the same file. We can quickly check dev/inode equality on
262    // unix-style systems.
263    #[cfg(unix)]
264    fn same_dev_inode(lhs: &Path, rhs: &Path) -> io::Result<bool> {
265        use std::os::unix::fs::MetadataExt;
266
267        let lhs = fs::metadata(lhs)?;
268        let rhs = fs::metadata(rhs)?;
269
270        Ok(lhs.dev() == rhs.dev() && lhs.ino() == rhs.ino())
271    }
272
273    #[cfg(not(unix))]
274    fn same_dev_inode(_lhs: &Path, _rhs: &Path) -> io::Result<bool> {
275        Ok(false)
276    }
277
278    if same_dev_inode(lhs, rhs)? {
279        return Ok(true);
280    }
281
282    // Next, check if the paths resolve to the same path.
283    let lhs = fs::canonicalize(lhs)?;
284    let rhs = fs::canonicalize(rhs)?;
285
286    if lhs == rhs {
287        return Ok(true);
288    }
289
290    // Next, see if the files have different lengths.
291    let lhs = fs::File::open(lhs)?;
292    let rhs = fs::File::open(rhs)?;
293
294    if lhs.metadata()?.len() != rhs.metadata()?.len() {
295        return Ok(false);
296    }
297
298    // Finally, check if the files have the same contents.
299    let mut lhs = io::BufReader::new(lhs).bytes();
300    let mut rhs = io::BufReader::new(rhs).bytes();
301
302    loop {
303        match (lhs.next(), rhs.next()) {
304            (None, None) => {
305                return Ok(true);
306            }
307            (Some(Ok(_)), None) | (None, Some(Ok(_))) => {
308                return Ok(false);
309            }
310            (Some(Ok(lhs_byte)), Some(Ok(rhs_byte))) => {
311                if lhs_byte != rhs_byte {
312                    return Ok(false);
313                }
314            }
315            (Some(Err(err)), _) | (_, Some(Err(err))) => {
316                return Err(err);
317            }
318        }
319    }
320}
321
322#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
323#[serde(tag = "version", content = "content", deny_unknown_fields)]
324enum VersionedPackageBuildManifest {
325    #[serde(rename = "1")]
326    Version1(PackageBuildManifestV1),
327}
328
329#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
330struct PackageBuildManifestV1 {
331    #[serde(rename = "/")]
332    external_contents: BTreeMap<String, String>,
333    #[serde(rename = "/meta/")]
334    far_contents: BTreeMap<String, String>,
335}
336
337#[cfg(test)]
338mod tests {
339    use super::*;
340    use crate::test::*;
341    use assert_matches::assert_matches;
342    use proptest::prelude::*;
343    use serde_json::json;
344    use std::fs::create_dir;
345
346    fn from_json_value(
347        value: serde_json::Value,
348    ) -> Result<PackageBuildManifest, PackageBuildManifestError> {
349        PackageBuildManifest::from_json(value.to_string().as_bytes())
350    }
351
352    #[test]
353    fn test_malformed_json() {
354        assert_matches!(
355            PackageBuildManifest::from_json("<invalid json document>".as_bytes()),
356            Err(PackageBuildManifestError::Json(err)) if err.is_syntax()
357        );
358    }
359
360    #[test]
361    fn test_invalid_version() {
362        assert_matches!(
363            from_json_value(json!({"version": "2", "content": {}})),
364            Err(PackageBuildManifestError::Json(err)) if err.is_data()
365        );
366    }
367
368    #[test]
369    fn test_invalid_resource_path() {
370        assert_matches!(
371            from_json_value(
372                json!(
373                    {"version": "1",
374                     "content":
375                     {"/meta/" :
376                      {"/starts-with-slash": "host-path"},
377                      "/": {
378                      }
379                     }
380                    }
381                )
382            ),
383            Err(PackageBuildManifestError::ResourcePath {
384                cause: fuchsia_url::ResourcePathError::PathStartsWithSlash,
385                path: s
386            }) if s == "/starts-with-slash"
387        );
388    }
389
390    #[test]
391    fn test_meta_dir_in_external() {
392        assert_matches!(
393            from_json_value(
394                json!(
395                    {"version": "1",
396                     "content":
397                     {"/meta/" : {},
398                      "/": {
399                          "meta/foo": "host-path"}
400                     }
401                    }
402                )
403            ),
404            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta/foo"
405        );
406    }
407
408    #[test]
409    fn test_meta_file_in_external() {
410        assert_matches!(
411            from_json_value(
412                json!({
413                    "version": "1",
414                    "content": {
415                        "/meta/" : {},
416                        "/": {
417                            "meta": "host-path"
418                        }
419                    }
420                })
421            ),
422            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta"
423        );
424    }
425
426    #[test]
427    fn test_file_dir_collision() {
428        for (path0, path1, expected_conflict) in [
429            ("foo", "foo/bar", "foo"),
430            ("foo/bar", "foo/bar/baz", "foo/bar"),
431            ("foo", "foo/bar/baz", "foo"),
432        ] {
433            let external = BTreeMap::from([
434                (path0.to_string(), String::new()),
435                (path1.to_string(), String::new()),
436            ]);
437            assert_matches!(
438                PackageBuildManifest::from_external_and_far_contents(external, BTreeMap::new()),
439                Err(PackageBuildManifestError::FileDirectoryCollision { path })
440                    if path == expected_conflict
441            );
442        }
443    }
444
445    #[test]
446    fn test_from_v1() {
447        assert_eq!(
448            from_json_value(json!(
449                {"version": "1",
450                 "content": {
451                     "/": {
452                         "this-path": "this-host-path",
453                         "that/path": "that/host/path"},
454                     "/meta/" : {
455                         "some-path": "some-host-path",
456                         "other/path": "other/host/path"}
457                 }
458                }
459            ))
460            .unwrap(),
461            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
462                external_contents: BTreeMap::from([
463                    ("this-path".to_string(), "this-host-path".to_string()),
464                    ("that/path".to_string(), "that/host/path".to_string()),
465                ]),
466                far_contents: BTreeMap::from([
467                    ("meta/some-path".to_string(), "some-host-path".to_string()),
468                    ("meta/other/path".to_string(), "other/host/path".to_string()),
469                ])
470            }))
471        );
472    }
473
474    #[test]
475    fn test_from_pm_fini() {
476        assert_eq!(
477            PackageBuildManifest::from_pm_fini(
478                "this-path=this-host-path\n\
479                 that/path=that/host/path\n\
480                 another/path=another/host=path\n
481                   with/white/space = host/white/space \n\n\
482                 meta/some-path=some-host-path\n\
483                 meta/other/path=other/host/path\n\
484                 meta/another/path=another/host=path\n\
485                 ignore lines without equals"
486                    .as_bytes()
487            )
488            .unwrap(),
489            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
490                external_contents: BTreeMap::from([
491                    ("this-path".to_string(), "this-host-path".to_string()),
492                    ("that/path".to_string(), "that/host/path".to_string()),
493                    ("another/path".to_string(), "another/host=path".to_string()),
494                    ("with/white/space".to_string(), "host/white/space".to_string()),
495                ]),
496                far_contents: BTreeMap::from([
497                    ("meta/some-path".to_string(), "some-host-path".to_string()),
498                    ("meta/other/path".to_string(), "other/host/path".to_string()),
499                    ("meta/another/path".to_string(), "another/host=path".to_string()),
500                ]),
501            })),
502        );
503    }
504
505    #[test]
506    fn test_from_pm_fini_empty() {
507        assert_eq!(
508            PackageBuildManifest::from_pm_fini("".as_bytes()).unwrap(),
509            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
510                external_contents: BTreeMap::new(),
511                far_contents: BTreeMap::new()
512            })),
513        );
514    }
515
516    #[test]
517    fn test_from_pm_fini_same_file_contents() {
518        let dir = tempfile::tempdir().unwrap();
519
520        let path = dir.path().join("path");
521        let same = dir.path().join("same");
522
523        fs::write(&path, b"hello world").unwrap();
524        fs::write(&same, b"hello world").unwrap();
525
526        let fini = format!(
527            "path={path}\n\
528             path={same}\n",
529            path = path.to_str().unwrap(),
530            same = same.to_str().unwrap(),
531        );
532
533        assert_eq!(
534            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
535            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
536                external_contents: BTreeMap::from([(
537                    "path".to_string(),
538                    path.to_str().unwrap().to_string()
539                ),]),
540                far_contents: BTreeMap::new(),
541            })),
542        );
543    }
544
545    #[test]
546    fn test_from_pm_fini_different_contents() {
547        let dir = tempfile::tempdir().unwrap();
548
549        let path = dir.path().join("path");
550        let different = dir.path().join("different");
551
552        fs::write(&path, b"hello world").unwrap();
553        fs::write(&different, b"different").unwrap();
554
555        let fini = format!(
556            "path={path}\n\
557             path={different}\n",
558            path = path.to_str().unwrap(),
559            different = different.to_str().unwrap()
560        );
561
562        assert_matches!(
563            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
564            Err(PackageBuildManifestError::DuplicateResourcePath { path }) if path == "path"
565        );
566    }
567
568    #[test]
569    fn test_from_dir() {
570        let dir = tempfile::tempdir().unwrap();
571
572        let blob1 = dir.path().join("blob1");
573        let blob2 = dir.path().join("blob2");
574        let meta_dir = dir.path().join("meta");
575        create_dir(&meta_dir).unwrap();
576
577        let meta_package = meta_dir.join("package");
578        let meta_data = meta_dir.join("data");
579
580        fs::write(blob1, b"blob1").unwrap();
581        fs::write(blob2, b"blob2").unwrap();
582        fs::write(meta_package, b"meta_package").unwrap();
583        fs::write(meta_data, b"meta_data").unwrap();
584
585        let creation_manifest = PackageBuildManifest::from_dir(dir.path()).unwrap();
586        let far_contents = creation_manifest.far_contents();
587        let external_contents = creation_manifest.external_contents();
588        assert!(far_contents.contains_key("meta/data"));
589        assert!(far_contents.contains_key("meta/package"));
590        assert!(external_contents.contains_key("blob1"));
591        assert!(external_contents.contains_key("blob2"));
592    }
593
594    #[test]
595    fn test_from_pm_fini_not_found() {
596        let dir = tempfile::tempdir().unwrap();
597
598        let path = dir.path().join("path");
599        let not_found = dir.path().join("not_found");
600
601        fs::write(&path, b"hello world").unwrap();
602
603        let fini = format!(
604            "path={path}\n\
605             path={not_found}\n",
606            path = path.to_str().unwrap(),
607            not_found = not_found.to_str().unwrap()
608        );
609
610        assert_matches!(
611            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
612            Err(PackageBuildManifestError::IoError(err)) if err.kind() == io::ErrorKind::NotFound
613        );
614    }
615
616    #[cfg(not(target_os = "fuchsia"))]
617    #[cfg(unix)]
618    #[test]
619    fn test_from_pm_fini_link() {
620        let dir = tempfile::tempdir().unwrap();
621
622        let path = dir.path().join("path");
623        let hard = dir.path().join("hard");
624        let sym = dir.path().join("symlink");
625
626        fs::write(&path, b"hello world").unwrap();
627        fs::hard_link(&path, &hard).unwrap();
628        std::os::unix::fs::symlink(&path, &sym).unwrap();
629
630        let fini = format!(
631            "path={path}\n\
632             path={hard}\n\
633             path={sym}\n",
634            path = path.to_str().unwrap(),
635            hard = hard.to_str().unwrap(),
636            sym = sym.to_str().unwrap(),
637        );
638
639        assert_eq!(
640            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
641            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
642                external_contents: BTreeMap::from([(
643                    "path".to_string(),
644                    path.to_str().unwrap().to_string()
645                ),]),
646                far_contents: BTreeMap::new(),
647            })),
648        );
649    }
650
651    proptest! {
652        #[test]
653        fn test_from_external_and_far_contents_does_not_modify_valid_maps(
654            ref external_resource_path in random_external_resource_path(),
655            ref external_host_path in ".{0,30}",
656            ref far_resource_path in random_far_resource_path(),
657            ref far_host_path in ".{0,30}"
658        ) {
659            let external_contents = BTreeMap::from([
660                (external_resource_path.to_string(), external_host_path.to_string()),
661            ]);
662            let far_resource_path = format!("meta/{far_resource_path}");
663            let far_contents = BTreeMap::from([
664                (far_resource_path, far_host_path.to_string()),
665            ]);
666
667            let creation_manifest = PackageBuildManifest::from_external_and_far_contents(
668                external_contents.clone(), far_contents.clone())
669                .unwrap();
670
671            prop_assert_eq!(creation_manifest.external_contents(), &external_contents);
672            prop_assert_eq!(creation_manifest.far_contents(), &far_contents);
673        }
674    }
675}
fuchsia_pkg/package_build_manifest.rs

fuchsia_pkg/
package_build_manifest.rs
