fuchsia_pkg/
package_build_manifest.rs

Help
1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::errors::PackageBuildManifestError;
6use fuchsia_url::validate_resource_path;
7use serde::{Deserialize, Serialize};
8use std::collections::{btree_map, BTreeMap, HashSet};
9use std::fs;
10use std::io::{self, Read};
11use std::path::Path;
12use walkdir::WalkDir;
13
14/// Package file list
15///
16/// A `PackageBuildManifest` lists the files that should be included in a Fuchsia package. Both
17/// `external_contents` and `far_contents` are maps from package resource paths in the to-be-created
18/// package to paths on the local filesystem. Package resource paths start with "meta/" if and only
19/// if they are in `far_contents`.
20#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
21#[serde(transparent)]
22pub struct PackageBuildManifest(VersionedPackageBuildManifest);
23
24impl PackageBuildManifest {
25    /// Creates a `PackageBuildManifest` from external and far contents maps.
26    ///
27    /// `external_contents` is a map from package resource paths to their locations
28    /// on the host filesystem. These are the files that will be listed in
29    /// `meta/contents`. Resource paths in `external_contents` must *not* start with
30    /// `meta/` or be exactly `meta`. Resource paths must not have file/directory collisions,
31    /// e.g. ["foo", "foo/bar"] have a file/directory collision at "foo", or they will be rejected.
32    /// `far_contents` is a map from package resource paths to their locations
33    /// on the host filesystem. These are the files that will be included bodily in the
34    /// package `meta.far` archive. Resource paths in `far_contents` must start with
35    /// `meta/`.
36    ///
37    /// # Examples
38    ///
39    /// ```
40    /// # use fuchsia_pkg::PackageBuildManifest;
41    /// # use maplit::btreemap;
42    /// let external_contents = btreemap! {
43    ///     "lib/mylib.so".to_string() => "build/system/path/mylib.so".to_string()
44    /// };
45    /// let far_contents = btreemap! {
46    ///     "meta/my_component_manifest.cm".to_string() =>
47    ///         "other/build/system/path/my_component_manifest.cm".to_string()
48    /// };
49    /// let creation_manifest =
50    ///     PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
51    ///         .unwrap();
52    /// ```
53    pub fn from_external_and_far_contents(
54        external_contents: BTreeMap<String, String>,
55        far_contents: BTreeMap<String, String>,
56    ) -> Result<Self, PackageBuildManifestError> {
57        for (resource_path, _) in external_contents.iter().chain(far_contents.iter()) {
58            validate_resource_path(resource_path).map_err(|e| {
59                PackageBuildManifestError::ResourcePath {
60                    cause: e,
61                    path: resource_path.to_string(),
62                }
63            })?;
64        }
65        let external_paths =
66            external_contents.keys().map(|path| path.as_str()).collect::<HashSet<_>>();
67        for resource_path in &external_paths {
68            if resource_path.starts_with("meta/") || resource_path.eq(&"meta") {
69                return Err(PackageBuildManifestError::ExternalContentInMetaDirectory {
70                    path: resource_path.to_string(),
71                });
72            }
73            for (i, _) in resource_path.match_indices('/') {
74                if external_paths.contains(&resource_path[..i]) {
75                    return Err(PackageBuildManifestError::FileDirectoryCollision {
76                        path: resource_path[..i].to_string(),
77                    });
78                }
79            }
80        }
81        for (resource_path, _) in far_contents.iter() {
82            if !resource_path.starts_with("meta/") {
83                return Err(PackageBuildManifestError::FarContentNotInMetaDirectory {
84                    path: resource_path.to_string(),
85                });
86            }
87        }
88        Ok(PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
89            external_contents,
90            far_contents,
91        })))
92    }
93
94    /// Deserializes a `PackageBuildManifest` from versioned json.
95    ///
96    /// # Examples
97    /// ```
98    /// # use fuchsia_pkg::PackageBuildManifest;
99    /// let json_string = r#"
100    /// {
101    ///   "version": "1",
102    ///   "content": {
103    ///     "/": {
104    ///       "lib/mylib.so": "build/system/path/mylib.so"
105    ///     },
106    ///    "/meta/": {
107    ///      "my_component_manifest.cml": "other/build/system/path/my_component_manifest.cml"
108    ///    }
109    /// }"#;
110    /// let creation_manifest = PackageBuildManifest::from_json(json_string.as_bytes());
111    /// ```
112    pub fn from_json<R: io::Read>(reader: R) -> Result<Self, PackageBuildManifestError> {
113        match serde_json::from_reader::<R, VersionedPackageBuildManifest>(reader)? {
114            VersionedPackageBuildManifest::Version1(v1) => PackageBuildManifest::from_v1(v1),
115        }
116    }
117
118    fn from_v1(v1: PackageBuildManifestV1) -> Result<Self, PackageBuildManifestError> {
119        let mut far_contents = BTreeMap::new();
120        // Validate package resource paths in far contents before "meta/" is prepended
121        // for better error messages.
122        for (resource_path, host_path) in v1.far_contents.into_iter() {
123            validate_resource_path(&resource_path).map_err(|e| {
124                PackageBuildManifestError::ResourcePath {
125                    cause: e,
126                    path: resource_path.to_string(),
127                }
128            })?;
129            far_contents.insert(format!("meta/{resource_path}"), host_path);
130        }
131        PackageBuildManifest::from_external_and_far_contents(v1.external_contents, far_contents)
132    }
133
134    pub fn from_dir(root: impl AsRef<Path>) -> Result<Self, PackageBuildManifestError> {
135        let root = root.as_ref();
136        let mut far_contents = BTreeMap::new();
137        let mut external_contents = BTreeMap::new();
138
139        for entry in WalkDir::new(root) {
140            let entry = entry?;
141            let path = entry.path();
142            let file_type = entry.file_type();
143            if file_type.is_dir() {
144                continue;
145            }
146            if !(file_type.is_file() || file_type.is_symlink()) {
147                return Err(PackageBuildManifestError::InvalidFileType {
148                    path: path.to_path_buf(),
149                });
150            }
151
152            let relative_path = path
153                .strip_prefix(root)?
154                .to_str()
155                .ok_or(PackageBuildManifestError::EmptyResourcePath)?;
156            let path =
157                path.to_str().ok_or(PackageBuildManifestError::EmptyResourcePath)?.to_owned();
158            if relative_path.starts_with("meta") {
159                far_contents.insert(relative_path.to_owned(), path);
160            } else {
161                external_contents.insert(relative_path.to_owned(), path);
162            }
163        }
164
165        PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
166    }
167
168    /// Create a `PackageBuildManifest` from a `pm-build`-style Fuchsia INI file (fini). fini is a
169    /// simple format where each line is an entry of `$PKG_PATH=$HOST_PATH`. This copies the
170    /// parsing algorithm from pm, where:
171    ///
172    /// * The $PKG_PATH is the string up to the first equals sign.
173    /// * If there is a duplicate entry, check if the two entries have the same file contents. If
174    ///   not, error out.
175    /// * Only check if the files exist upon duplicate entry.
176    /// * Ignores lines without an '=' in it.
177    ///
178    /// Note: This functionality exists only to ease the migration from `pm build`. This will be
179    /// removed once there are no more users of `pm build`-style manifests.
180    ///
181    /// # Examples
182    ///
183    /// ```
184    /// # use fuchsia_pkg::PackageBuildManifest;
185    /// let fini_string = "\
186    ///     lib/mylib.so=build/system/path/mylib.so\n\
187    ///     meta/my_component_manifest.cml=other/build/system/path/my_component_manifest.cml\n";
188    ///
189    /// let creation_manifest = PackageBuildManifest::from_pm_fini(fini_string.as_bytes()).unwrap();
190    /// ```
191    pub fn from_pm_fini<R: io::BufRead>(mut reader: R) -> Result<Self, PackageBuildManifestError> {
192        let mut external_contents = BTreeMap::new();
193        let mut far_contents = BTreeMap::new();
194
195        let mut buf = String::new();
196        while reader.read_line(&mut buf)? != 0 {
197            let line = buf.trim();
198            if line.is_empty() {
199                buf.clear();
200                continue;
201            }
202
203            // pm's build manifest finds the first '='. If one doesn't exist the line is ignored.
204            let pos = if let Some(pos) = line.find('=') {
205                pos
206            } else {
207                buf.clear();
208                continue;
209            };
210
211            let package_path = line[..pos].trim().to_string();
212            let host_path = line[pos + 1..].trim().to_string();
213
214            let entry = if package_path.starts_with("meta/") {
215                far_contents.entry(package_path)
216            } else {
217                external_contents.entry(package_path)
218            };
219
220            match entry {
221                btree_map::Entry::Vacant(entry) => {
222                    entry.insert(host_path);
223                }
224                btree_map::Entry::Occupied(entry) => {
225                    // `pm build` manifests allow for duplicate entries, as long as they point to
226                    // the same file.
227                    if !same_file_contents(Path::new(&entry.get()), Path::new(&host_path))? {
228                        return Err(PackageBuildManifestError::DuplicateResourcePath {
229                            path: entry.key().clone(),
230                        });
231                    }
232                }
233            }
234
235            buf.clear();
236        }
237
238        Self::from_external_and_far_contents(external_contents, far_contents)
239    }
240
241    /// `external_contents` lists the blobs that make up the meta/contents file
242    pub fn external_contents(&self) -> &BTreeMap<String, String> {
243        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
244        &manifest.external_contents
245    }
246
247    /// `far_contents` lists the files to be included bodily in the meta.far
248    pub fn far_contents(&self) -> &BTreeMap<String, String> {
249        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
250        &manifest.far_contents
251    }
252}
253
254// It's possible for the same host file to be discovered through multiple paths. This is allowed as
255// long as the files have the same file contents.
256fn same_file_contents(lhs: &Path, rhs: &Path) -> io::Result<bool> {
257    // First, check if the two paths are the same.
258    if lhs == rhs {
259        return Ok(true);
260    }
261
262    // Next, check if the paths point at the same file. We can quickly check dev/inode equality on
263    // unix-style systems.
264    #[cfg(unix)]
265    fn same_dev_inode(lhs: &Path, rhs: &Path) -> io::Result<bool> {
266        use std::os::unix::fs::MetadataExt;
267
268        let lhs = fs::metadata(lhs)?;
269        let rhs = fs::metadata(rhs)?;
270
271        Ok(lhs.dev() == rhs.dev() && lhs.ino() == rhs.ino())
272    }
273
274    #[cfg(not(unix))]
275    fn same_dev_inode(_lhs: &Path, _rhs: &Path) -> io::Result<bool> {
276        Ok(false)
277    }
278
279    if same_dev_inode(lhs, rhs)? {
280        return Ok(true);
281    }
282
283    // Next, check if the paths resolve to the same path.
284    let lhs = fs::canonicalize(lhs)?;
285    let rhs = fs::canonicalize(rhs)?;
286
287    if lhs == rhs {
288        return Ok(true);
289    }
290
291    // Next, see if the files have different lengths.
292    let lhs = fs::File::open(lhs)?;
293    let rhs = fs::File::open(rhs)?;
294
295    if lhs.metadata()?.len() != rhs.metadata()?.len() {
296        return Ok(false);
297    }
298
299    // Finally, check if the files have the same contents.
300    let mut lhs = io::BufReader::new(lhs).bytes();
301    let mut rhs = io::BufReader::new(rhs).bytes();
302
303    loop {
304        match (lhs.next(), rhs.next()) {
305            (None, None) => {
306                return Ok(true);
307            }
308            (Some(Ok(_)), None) | (None, Some(Ok(_))) => {
309                return Ok(false);
310            }
311            (Some(Ok(lhs_byte)), Some(Ok(rhs_byte))) => {
312                if lhs_byte != rhs_byte {
313                    return Ok(false);
314                }
315            }
316            (Some(Err(err)), _) | (_, Some(Err(err))) => {
317                return Err(err);
318            }
319        }
320    }
321}
322
323#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
324#[serde(tag = "version", content = "content", deny_unknown_fields)]
325enum VersionedPackageBuildManifest {
326    #[serde(rename = "1")]
327    Version1(PackageBuildManifestV1),
328}
329
330#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
331struct PackageBuildManifestV1 {
332    #[serde(rename = "/")]
333    external_contents: BTreeMap<String, String>,
334    #[serde(rename = "/meta/")]
335    far_contents: BTreeMap<String, String>,
336}
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341    use crate::test::*;
342    use assert_matches::assert_matches;
343    use fuchsia_url::errors::ResourcePathError::PathStartsWithSlash;
344    use maplit::btreemap;
345    use proptest::prelude::*;
346    use serde_json::json;
347    use std::fs::create_dir;
348
349    fn from_json_value(
350        value: serde_json::Value,
351    ) -> Result<PackageBuildManifest, PackageBuildManifestError> {
352        PackageBuildManifest::from_json(value.to_string().as_bytes())
353    }
354
355    #[test]
356    fn test_malformed_json() {
357        assert_matches!(
358            PackageBuildManifest::from_json("<invalid json document>".as_bytes()),
359            Err(PackageBuildManifestError::Json(err)) if err.is_syntax()
360        );
361    }
362
363    #[test]
364    fn test_invalid_version() {
365        assert_matches!(
366            from_json_value(json!({"version": "2", "content": {}})),
367            Err(PackageBuildManifestError::Json(err)) if err.is_data()
368        );
369    }
370
371    #[test]
372    fn test_invalid_resource_path() {
373        assert_matches!(
374            from_json_value(
375                json!(
376                    {"version": "1",
377                     "content":
378                     {"/meta/" :
379                      {"/starts-with-slash": "host-path"},
380                      "/": {
381                      }
382                     }
383                    }
384                )
385            ),
386            Err(PackageBuildManifestError::ResourcePath {
387                cause: PathStartsWithSlash,
388                path: s
389            }) if s == "/starts-with-slash"
390        );
391    }
392
393    #[test]
394    fn test_meta_dir_in_external() {
395        assert_matches!(
396            from_json_value(
397                json!(
398                    {"version": "1",
399                     "content":
400                     {"/meta/" : {},
401                      "/": {
402                          "meta/foo": "host-path"}
403                     }
404                    }
405                )
406            ),
407            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta/foo"
408        );
409    }
410
411    #[test]
412    fn test_meta_file_in_external() {
413        assert_matches!(
414            from_json_value(
415                json!({
416                    "version": "1",
417                    "content": {
418                        "/meta/" : {},
419                        "/": {
420                            "meta": "host-path"
421                        }
422                    }
423                })
424            ),
425            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta"
426        );
427    }
428
429    #[test]
430    fn test_file_dir_collision() {
431        for (path0, path1, expected_conflict) in [
432            ("foo", "foo/bar", "foo"),
433            ("foo/bar", "foo/bar/baz", "foo/bar"),
434            ("foo", "foo/bar/baz", "foo"),
435        ] {
436            let external = btreemap! {
437                path0.to_string() => String::new(),
438                path1.to_string() => String::new(),
439            };
440            assert_matches!(
441                PackageBuildManifest::from_external_and_far_contents(external, BTreeMap::new()),
442                Err(PackageBuildManifestError::FileDirectoryCollision { path })
443                    if path == expected_conflict
444            );
445        }
446    }
447
448    #[test]
449    fn test_from_v1() {
450        assert_eq!(
451            from_json_value(json!(
452                {"version": "1",
453                 "content": {
454                     "/": {
455                         "this-path": "this-host-path",
456                         "that/path": "that/host/path"},
457                     "/meta/" : {
458                         "some-path": "some-host-path",
459                         "other/path": "other/host/path"}
460                 }
461                }
462            ))
463            .unwrap(),
464            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
465                external_contents: btreemap! {
466                    "this-path".to_string() => "this-host-path".to_string(),
467                    "that/path".to_string() => "that/host/path".to_string()
468                },
469                far_contents: btreemap! {
470                    "meta/some-path".to_string() => "some-host-path".to_string(),
471                    "meta/other/path".to_string() => "other/host/path".to_string()
472                }
473            }))
474        );
475    }
476
477    #[test]
478    fn test_from_pm_fini() {
479        assert_eq!(
480            PackageBuildManifest::from_pm_fini(
481                "this-path=this-host-path\n\
482                 that/path=that/host/path\n\
483                 another/path=another/host=path\n
484                   with/white/space = host/white/space \n\n\
485                 meta/some-path=some-host-path\n\
486                 meta/other/path=other/host/path\n\
487                 meta/another/path=another/host=path\n\
488                 ignore lines without equals"
489                    .as_bytes()
490            )
491            .unwrap(),
492            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
493                external_contents: btreemap! {
494                    "this-path".to_string() => "this-host-path".to_string(),
495                    "that/path".to_string() => "that/host/path".to_string(),
496                    "another/path".to_string() => "another/host=path".to_string(),
497                    "with/white/space".to_string() => "host/white/space".to_string(),
498                },
499                far_contents: btreemap! {
500                    "meta/some-path".to_string() => "some-host-path".to_string(),
501                    "meta/other/path".to_string() => "other/host/path".to_string(),
502                    "meta/another/path".to_string() => "another/host=path".to_string(),
503                },
504            })),
505        );
506    }
507
508    #[test]
509    fn test_from_pm_fini_empty() {
510        assert_eq!(
511            PackageBuildManifest::from_pm_fini("".as_bytes()).unwrap(),
512            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
513                external_contents: btreemap! {},
514                far_contents: btreemap! {}
515            })),
516        );
517    }
518
519    #[test]
520    fn test_from_pm_fini_same_file_contents() {
521        let dir = tempfile::tempdir().unwrap();
522
523        let path = dir.path().join("path");
524        let same = dir.path().join("same");
525
526        fs::write(&path, b"hello world").unwrap();
527        fs::write(&same, b"hello world").unwrap();
528
529        let fini = format!(
530            "path={path}\n\
531             path={same}\n",
532            path = path.to_str().unwrap(),
533            same = same.to_str().unwrap(),
534        );
535
536        assert_eq!(
537            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
538            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
539                external_contents: btreemap! {
540                    "path".to_string() => path.to_str().unwrap().to_string(),
541                },
542                far_contents: btreemap! {},
543            })),
544        );
545    }
546
547    #[test]
548    fn test_from_pm_fini_different_contents() {
549        let dir = tempfile::tempdir().unwrap();
550
551        let path = dir.path().join("path");
552        let different = dir.path().join("different");
553
554        fs::write(&path, b"hello world").unwrap();
555        fs::write(&different, b"different").unwrap();
556
557        let fini = format!(
558            "path={path}\n\
559             path={different}\n",
560            path = path.to_str().unwrap(),
561            different = different.to_str().unwrap()
562        );
563
564        assert_matches!(
565            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
566            Err(PackageBuildManifestError::DuplicateResourcePath { path }) if path == "path"
567        );
568    }
569
570    #[test]
571    fn test_from_dir() {
572        let dir = tempfile::tempdir().unwrap();
573
574        let blob1 = dir.path().join("blob1");
575        let blob2 = dir.path().join("blob2");
576        let meta_dir = dir.path().join("meta");
577        create_dir(&meta_dir).unwrap();
578
579        let meta_package = meta_dir.join("package");
580        let meta_data = meta_dir.join("data");
581
582        fs::write(blob1, b"blob1").unwrap();
583        fs::write(blob2, b"blob2").unwrap();
584        fs::write(meta_package, b"meta_package").unwrap();
585        fs::write(meta_data, b"meta_data").unwrap();
586
587        let creation_manifest = PackageBuildManifest::from_dir(dir.path()).unwrap();
588        let far_contents = creation_manifest.far_contents();
589        let external_contents = creation_manifest.external_contents();
590        assert!(far_contents.contains_key("meta/data"));
591        assert!(far_contents.contains_key("meta/package"));
592        assert!(external_contents.contains_key("blob1"));
593        assert!(external_contents.contains_key("blob2"));
594    }
595
596    #[test]
597    fn test_from_pm_fini_not_found() {
598        let dir = tempfile::tempdir().unwrap();
599
600        let path = dir.path().join("path");
601        let not_found = dir.path().join("not_found");
602
603        fs::write(&path, b"hello world").unwrap();
604
605        let fini = format!(
606            "path={path}\n\
607             path={not_found}\n",
608            path = path.to_str().unwrap(),
609            not_found = not_found.to_str().unwrap()
610        );
611
612        assert_matches!(
613            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
614            Err(PackageBuildManifestError::IoError(err)) if err.kind() == io::ErrorKind::NotFound
615        );
616    }
617
618    #[cfg(not(target_os = "fuchsia"))]
619    #[cfg(unix)]
620    #[test]
621    fn test_from_pm_fini_link() {
622        let dir = tempfile::tempdir().unwrap();
623
624        let path = dir.path().join("path");
625        let hard = dir.path().join("hard");
626        let sym = dir.path().join("symlink");
627
628        fs::write(&path, b"hello world").unwrap();
629        fs::hard_link(&path, &hard).unwrap();
630        std::os::unix::fs::symlink(&path, &sym).unwrap();
631
632        let fini = format!(
633            "path={path}\n\
634             path={hard}\n\
635             path={sym}\n",
636            path = path.to_str().unwrap(),
637            hard = hard.to_str().unwrap(),
638            sym = sym.to_str().unwrap(),
639        );
640
641        assert_eq!(
642            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
643            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
644                external_contents: btreemap! {
645                    "path".to_string() => path.to_str().unwrap().to_string(),
646                },
647                far_contents: btreemap! {},
648            })),
649        );
650    }
651
652    proptest! {
653        #[test]
654        fn test_from_external_and_far_contents_does_not_modify_valid_maps(
655            ref external_resource_path in random_external_resource_path(),
656            ref external_host_path in ".{0,30}",
657            ref far_resource_path in random_far_resource_path(),
658            ref far_host_path in ".{0,30}"
659        ) {
660            let external_contents = btreemap! {
661                external_resource_path.to_string() => external_host_path.to_string()
662            };
663            let far_resource_path = format!("meta/{far_resource_path}");
664            let far_contents = btreemap! {
665                far_resource_path => far_host_path.to_string()
666            };
667
668            let creation_manifest = PackageBuildManifest::from_external_and_far_contents(
669                external_contents.clone(), far_contents.clone())
670                .unwrap();
671
672            prop_assert_eq!(creation_manifest.external_contents(), &external_contents);
673            prop_assert_eq!(creation_manifest.far_contents(), &far_contents);
674        }
675    }
676}
fuchsia_pkg/package_build_manifest.rs

fuchsia_pkg/
package_build_manifest.rs
