fuchsia_pkg/
package_build_manifest.rs

1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::errors::PackageBuildManifestError;
6use buf_read_ext::BufReadExt as _;
7use serde::{Deserialize, Serialize};
8use std::collections::{BTreeMap, HashSet, btree_map};
9use std::fs;
10use std::io::{self, Read};
11use std::path::Path;
12use walkdir::WalkDir;
13
14/// Package file list
15///
16/// A `PackageBuildManifest` lists the files that should be included in a Fuchsia package. Both
17/// `external_contents` and `far_contents` are maps from package resource paths in the to-be-created
18/// package to paths on the local filesystem. Package resource paths start with "meta/" if and only
19/// if they are in `far_contents`.
20#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
21#[serde(transparent)]
22pub struct PackageBuildManifest(VersionedPackageBuildManifest);
23
24impl PackageBuildManifest {
25    /// Creates a `PackageBuildManifest` from external and far contents maps.
26    ///
27    /// `external_contents` is a map from package resource paths to their locations
28    /// on the host filesystem. These are the files that will be listed in
29    /// `meta/contents`. Resource paths in `external_contents` must *not* start with
30    /// `meta/` or be exactly `meta`. Resource paths must not have file/directory collisions,
31    /// e.g. ["foo", "foo/bar"] have a file/directory collision at "foo", or they will be rejected.
32    /// `far_contents` is a map from package resource paths to their locations
33    /// on the host filesystem. These are the files that will be included bodily in the
34    /// package `meta.far` archive. Resource paths in `far_contents` must start with
35    /// `meta/`.
36    ///
37    /// # Examples
38    ///
39    /// ```
40    /// # use fuchsia_pkg::PackageBuildManifest;
41    /// # use std::collections::BTreeMap;
42    /// let external_contents = BTreeMap::from([
43    ///     ("lib/mylib.so".to_string(), "build/system/path/mylib.so".to_string()),
44    /// ]);
45    /// let far_contents = BTreeMap::from([
46    ///     ("meta/my_component_manifest.cm".to_string(),
47    ///         "other/build/system/path/my_component_manifest.cm".to_string()),
48    /// ]);
49    /// let creation_manifest =
50    ///     PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
51    ///         .unwrap();
52    /// ```
53    pub fn from_external_and_far_contents(
54        external_contents: BTreeMap<String, String>,
55        far_contents: BTreeMap<String, String>,
56    ) -> Result<Self, PackageBuildManifestError> {
57        for (resource_path, _) in external_contents.iter().chain(far_contents.iter()) {
58            fuchsia_url::Resource::validate_str(resource_path).map_err(|e| {
59                PackageBuildManifestError::ResourcePath {
60                    cause: e,
61                    path: resource_path.to_string(),
62                }
63            })?;
64        }
65        let external_paths =
66            external_contents.keys().map(|path| path.as_str()).collect::<HashSet<_>>();
67        for resource_path in &external_paths {
68            if resource_path.starts_with("meta/") || resource_path.eq(&"meta") {
69                return Err(PackageBuildManifestError::ExternalContentInMetaDirectory {
70                    path: resource_path.to_string(),
71                });
72            }
73            for (i, _) in resource_path.match_indices('/') {
74                if external_paths.contains(&resource_path[..i]) {
75                    return Err(PackageBuildManifestError::FileDirectoryCollision {
76                        path: resource_path[..i].to_string(),
77                    });
78                }
79            }
80        }
81        for (resource_path, _) in far_contents.iter() {
82            if !resource_path.starts_with("meta/") {
83                return Err(PackageBuildManifestError::FarContentNotInMetaDirectory {
84                    path: resource_path.to_string(),
85                });
86            }
87        }
88        Ok(PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
89            external_contents,
90            far_contents,
91        })))
92    }
93
94    /// Deserializes a `PackageBuildManifest` from versioned json.
95    ///
96    /// # Examples
97    /// ```
98    /// # use fuchsia_pkg::PackageBuildManifest;
99    /// let json_string = r#"
100    /// {
101    ///   "version": "1",
102    ///   "content": {
103    ///     "/": {
104    ///       "lib/mylib.so": "build/system/path/mylib.so"
105    ///     },
106    ///    "/meta/": {
107    ///      "my_component_manifest.cml": "other/build/system/path/my_component_manifest.cml"
108    ///    }
109    /// }"#;
110    /// let creation_manifest = PackageBuildManifest::from_json(json_string.as_bytes());
111    /// ```
112    pub fn from_json<R: io::Read>(reader: R) -> Result<Self, PackageBuildManifestError> {
113        match serde_json::from_reader::<R, VersionedPackageBuildManifest>(reader)? {
114            VersionedPackageBuildManifest::Version1(v1) => PackageBuildManifest::from_v1(v1),
115        }
116    }
117
118    fn from_v1(v1: PackageBuildManifestV1) -> Result<Self, PackageBuildManifestError> {
119        let mut far_contents = BTreeMap::new();
120        // Validate package resource paths in far contents before "meta/" is prepended
121        // for better error messages.
122        for (resource_path, host_path) in v1.far_contents.into_iter() {
123            fuchsia_url::Resource::validate_str(&resource_path).map_err(|e| {
124                PackageBuildManifestError::ResourcePath {
125                    cause: e,
126                    path: resource_path.to_string(),
127                }
128            })?;
129            far_contents.insert(format!("meta/{resource_path}"), host_path);
130        }
131        PackageBuildManifest::from_external_and_far_contents(v1.external_contents, far_contents)
132    }
133
134    pub fn from_dir(root: impl AsRef<Path>) -> Result<Self, PackageBuildManifestError> {
135        let root = root.as_ref();
136        let mut far_contents = BTreeMap::new();
137        let mut external_contents = BTreeMap::new();
138
139        for entry in WalkDir::new(root) {
140            let entry = entry?;
141            let path = entry.path();
142            let file_type = entry.file_type();
143            if file_type.is_dir() {
144                continue;
145            }
146            if !(file_type.is_file() || file_type.is_symlink()) {
147                return Err(PackageBuildManifestError::InvalidFileType {
148                    path: path.to_path_buf(),
149                });
150            }
151
152            let relative_path = path
153                .strip_prefix(root)?
154                .to_str()
155                .ok_or(PackageBuildManifestError::EmptyResourcePath)?;
156            let path =
157                path.to_str().ok_or(PackageBuildManifestError::EmptyResourcePath)?.to_owned();
158            if relative_path.starts_with("meta") {
159                far_contents.insert(relative_path.to_owned(), path);
160            } else {
161                external_contents.insert(relative_path.to_owned(), path);
162            }
163        }
164
165        PackageBuildManifest::from_external_and_far_contents(external_contents, far_contents)
166    }
167
168    /// Create a `PackageBuildManifest` from a `pm-build`-style Fuchsia INI file (fini). fini is a
169    /// simple format where each line is an entry of `$PKG_PATH=$HOST_PATH`. This copies the
170    /// parsing algorithm from pm, where:
171    ///
172    /// * The $PKG_PATH is the string up to the first equals sign.
173    /// * If there is a duplicate entry, check if the two entries have the same file contents. If
174    ///   not, error out.
175    /// * Only check if the files exist upon duplicate entry.
176    /// * Ignores lines without an '=' in it.
177    ///
178    /// Note: This functionality exists only to ease the migration from `pm build`. This will be
179    /// removed once there are no more users of `pm build`-style manifests.
180    ///
181    /// # Examples
182    ///
183    /// ```
184    /// # use fuchsia_pkg::PackageBuildManifest;
185    /// let fini_string = "\
186    ///     lib/mylib.so=build/system/path/mylib.so\n\
187    ///     meta/my_component_manifest.cml=other/build/system/path/my_component_manifest.cml\n";
188    ///
189    /// let creation_manifest = PackageBuildManifest::from_pm_fini(fini_string.as_bytes()).unwrap();
190    /// ```
191    pub fn from_pm_fini<R: io::BufRead>(mut reader: R) -> Result<Self, PackageBuildManifestError> {
192        let mut external_contents = BTreeMap::new();
193        let mut far_contents = BTreeMap::new();
194
195        let mut lines = reader.lending_lines();
196        while let Some(line) = lines.next() {
197            let line = line?.trim();
198            if line.is_empty() {
199                continue;
200            }
201
202            // pm's build manifest finds the first '='. If one doesn't exist the line is ignored.
203            let pos = if let Some(pos) = line.find('=') {
204                pos
205            } else {
206                continue;
207            };
208
209            let package_path = line[..pos].trim().to_string();
210            let host_path = line[pos + 1..].trim().to_string();
211
212            let entry = if package_path.starts_with("meta/") {
213                far_contents.entry(package_path)
214            } else {
215                external_contents.entry(package_path)
216            };
217
218            match entry {
219                btree_map::Entry::Vacant(entry) => {
220                    entry.insert(host_path);
221                }
222                btree_map::Entry::Occupied(entry) => {
223                    // `pm build` manifests allow for duplicate entries, as long as they point to
224                    // the same file.
225                    if !same_file_contents(Path::new(&entry.get()), Path::new(&host_path))? {
226                        return Err(PackageBuildManifestError::DuplicateResourcePath {
227                            path: entry.key().clone(),
228                        });
229                    }
230                }
231            }
232        }
233
234        Self::from_external_and_far_contents(external_contents, far_contents)
235    }
236
237    /// `external_contents` lists the blobs that make up the meta/contents file
238    pub fn external_contents(&self) -> &BTreeMap<String, String> {
239        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
240        &manifest.external_contents
241    }
242
243    /// `far_contents` lists the files to be included bodily in the meta.far
244    pub fn far_contents(&self) -> &BTreeMap<String, String> {
245        let VersionedPackageBuildManifest::Version1(manifest) = &self.0;
246        &manifest.far_contents
247    }
248}
249
250// It's possible for the same host file to be discovered through multiple paths. This is allowed as
251// long as the files have the same file contents.
252fn same_file_contents(lhs: &Path, rhs: &Path) -> io::Result<bool> {
253    // First, check if the two paths are the same.
254    if lhs == rhs {
255        return Ok(true);
256    }
257
258    // Next, check if the paths point at the same file. We can quickly check dev/inode equality on
259    // unix-style systems.
260    #[cfg(unix)]
261    fn same_dev_inode(lhs: &Path, rhs: &Path) -> io::Result<bool> {
262        use std::os::unix::fs::MetadataExt;
263
264        let lhs = fs::metadata(lhs)?;
265        let rhs = fs::metadata(rhs)?;
266
267        Ok(lhs.dev() == rhs.dev() && lhs.ino() == rhs.ino())
268    }
269
270    #[cfg(not(unix))]
271    fn same_dev_inode(_lhs: &Path, _rhs: &Path) -> io::Result<bool> {
272        Ok(false)
273    }
274
275    if same_dev_inode(lhs, rhs)? {
276        return Ok(true);
277    }
278
279    // Next, check if the paths resolve to the same path.
280    let lhs = fs::canonicalize(lhs)?;
281    let rhs = fs::canonicalize(rhs)?;
282
283    if lhs == rhs {
284        return Ok(true);
285    }
286
287    // Next, see if the files have different lengths.
288    let lhs = fs::File::open(lhs)?;
289    let rhs = fs::File::open(rhs)?;
290
291    if lhs.metadata()?.len() != rhs.metadata()?.len() {
292        return Ok(false);
293    }
294
295    // Finally, check if the files have the same contents.
296    let mut lhs = io::BufReader::new(lhs).bytes();
297    let mut rhs = io::BufReader::new(rhs).bytes();
298
299    loop {
300        match (lhs.next(), rhs.next()) {
301            (None, None) => {
302                return Ok(true);
303            }
304            (Some(Ok(_)), None) | (None, Some(Ok(_))) => {
305                return Ok(false);
306            }
307            (Some(Ok(lhs_byte)), Some(Ok(rhs_byte))) => {
308                if lhs_byte != rhs_byte {
309                    return Ok(false);
310                }
311            }
312            (Some(Err(err)), _) | (_, Some(Err(err))) => {
313                return Err(err);
314            }
315        }
316    }
317}
318
319#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
320#[serde(tag = "version", content = "content", deny_unknown_fields)]
321enum VersionedPackageBuildManifest {
322    #[serde(rename = "1")]
323    Version1(PackageBuildManifestV1),
324}
325
326#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
327struct PackageBuildManifestV1 {
328    #[serde(rename = "/")]
329    external_contents: BTreeMap<String, String>,
330    #[serde(rename = "/meta/")]
331    far_contents: BTreeMap<String, String>,
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337    use crate::test::*;
338    use assert_matches::assert_matches;
339    use proptest::prelude::*;
340    use serde_json::json;
341    use std::fs::create_dir;
342
343    fn from_json_value(
344        value: serde_json::Value,
345    ) -> Result<PackageBuildManifest, PackageBuildManifestError> {
346        PackageBuildManifest::from_json(value.to_string().as_bytes())
347    }
348
349    #[test]
350    fn test_malformed_json() {
351        assert_matches!(
352            PackageBuildManifest::from_json("<invalid json document>".as_bytes()),
353            Err(PackageBuildManifestError::Json(err)) if err.is_syntax()
354        );
355    }
356
357    #[test]
358    fn test_invalid_version() {
359        assert_matches!(
360            from_json_value(json!({"version": "2", "content": {}})),
361            Err(PackageBuildManifestError::Json(err)) if err.is_data()
362        );
363    }
364
365    #[test]
366    fn test_invalid_resource_path() {
367        assert_matches!(
368            from_json_value(
369                json!(
370                    {"version": "1",
371                     "content":
372                     {"/meta/" :
373                      {"/starts-with-slash": "host-path"},
374                      "/": {
375                      }
376                     }
377                    }
378                )
379            ),
380            Err(PackageBuildManifestError::ResourcePath {
381                cause: fuchsia_url::ResourcePathError::PathStartsWithSlash,
382                path: s
383            }) if s == "/starts-with-slash"
384        );
385    }
386
387    #[test]
388    fn test_meta_dir_in_external() {
389        assert_matches!(
390            from_json_value(
391                json!(
392                    {"version": "1",
393                     "content":
394                     {"/meta/" : {},
395                      "/": {
396                          "meta/foo": "host-path"}
397                     }
398                    }
399                )
400            ),
401            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta/foo"
402        );
403    }
404
405    #[test]
406    fn test_meta_file_in_external() {
407        assert_matches!(
408            from_json_value(
409                json!({
410                    "version": "1",
411                    "content": {
412                        "/meta/" : {},
413                        "/": {
414                            "meta": "host-path"
415                        }
416                    }
417                })
418            ),
419            Err(PackageBuildManifestError::ExternalContentInMetaDirectory{path: s}) if s == "meta"
420        );
421    }
422
423    #[test]
424    fn test_file_dir_collision() {
425        for (path0, path1, expected_conflict) in [
426            ("foo", "foo/bar", "foo"),
427            ("foo/bar", "foo/bar/baz", "foo/bar"),
428            ("foo", "foo/bar/baz", "foo"),
429        ] {
430            let external = BTreeMap::from([
431                (path0.to_string(), String::new()),
432                (path1.to_string(), String::new()),
433            ]);
434            assert_matches!(
435                PackageBuildManifest::from_external_and_far_contents(external, BTreeMap::new()),
436                Err(PackageBuildManifestError::FileDirectoryCollision { path })
437                    if path == expected_conflict
438            );
439        }
440    }
441
442    #[test]
443    fn test_from_v1() {
444        assert_eq!(
445            from_json_value(json!(
446                {"version": "1",
447                 "content": {
448                     "/": {
449                         "this-path": "this-host-path",
450                         "that/path": "that/host/path"},
451                     "/meta/" : {
452                         "some-path": "some-host-path",
453                         "other/path": "other/host/path"}
454                 }
455                }
456            ))
457            .unwrap(),
458            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
459                external_contents: BTreeMap::from([
460                    ("this-path".to_string(), "this-host-path".to_string()),
461                    ("that/path".to_string(), "that/host/path".to_string()),
462                ]),
463                far_contents: BTreeMap::from([
464                    ("meta/some-path".to_string(), "some-host-path".to_string()),
465                    ("meta/other/path".to_string(), "other/host/path".to_string()),
466                ])
467            }))
468        );
469    }
470
471    #[test]
472    fn test_from_pm_fini() {
473        assert_eq!(
474            PackageBuildManifest::from_pm_fini(
475                "this-path=this-host-path\n\
476                 that/path=that/host/path\n\
477                 another/path=another/host=path\n
478                   with/white/space = host/white/space \n\n\
479                 meta/some-path=some-host-path\n\
480                 meta/other/path=other/host/path\n\
481                 meta/another/path=another/host=path\n\
482                 ignore lines without equals"
483                    .as_bytes()
484            )
485            .unwrap(),
486            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
487                external_contents: BTreeMap::from([
488                    ("this-path".to_string(), "this-host-path".to_string()),
489                    ("that/path".to_string(), "that/host/path".to_string()),
490                    ("another/path".to_string(), "another/host=path".to_string()),
491                    ("with/white/space".to_string(), "host/white/space".to_string()),
492                ]),
493                far_contents: BTreeMap::from([
494                    ("meta/some-path".to_string(), "some-host-path".to_string()),
495                    ("meta/other/path".to_string(), "other/host/path".to_string()),
496                    ("meta/another/path".to_string(), "another/host=path".to_string()),
497                ]),
498            })),
499        );
500    }
501
502    #[test]
503    fn test_from_pm_fini_empty() {
504        assert_eq!(
505            PackageBuildManifest::from_pm_fini("".as_bytes()).unwrap(),
506            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
507                external_contents: BTreeMap::new(),
508                far_contents: BTreeMap::new()
509            })),
510        );
511    }
512
513    #[test]
514    fn test_from_pm_fini_same_file_contents() {
515        let dir = tempfile::tempdir().unwrap();
516
517        let path = dir.path().join("path");
518        let same = dir.path().join("same");
519
520        fs::write(&path, b"hello world").unwrap();
521        fs::write(&same, b"hello world").unwrap();
522
523        let fini = format!(
524            "path={path}\n\
525             path={same}\n",
526            path = path.to_str().unwrap(),
527            same = same.to_str().unwrap(),
528        );
529
530        assert_eq!(
531            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
532            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
533                external_contents: BTreeMap::from([(
534                    "path".to_string(),
535                    path.to_str().unwrap().to_string()
536                ),]),
537                far_contents: BTreeMap::new(),
538            })),
539        );
540    }
541
542    #[test]
543    fn test_from_pm_fini_different_contents() {
544        let dir = tempfile::tempdir().unwrap();
545
546        let path = dir.path().join("path");
547        let different = dir.path().join("different");
548
549        fs::write(&path, b"hello world").unwrap();
550        fs::write(&different, b"different").unwrap();
551
552        let fini = format!(
553            "path={path}\n\
554             path={different}\n",
555            path = path.to_str().unwrap(),
556            different = different.to_str().unwrap()
557        );
558
559        assert_matches!(
560            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
561            Err(PackageBuildManifestError::DuplicateResourcePath { path }) if path == "path"
562        );
563    }
564
565    #[test]
566    fn test_from_dir() {
567        let dir = tempfile::tempdir().unwrap();
568
569        let blob1 = dir.path().join("blob1");
570        let blob2 = dir.path().join("blob2");
571        let meta_dir = dir.path().join("meta");
572        create_dir(&meta_dir).unwrap();
573
574        let meta_package = meta_dir.join("package");
575        let meta_data = meta_dir.join("data");
576
577        fs::write(blob1, b"blob1").unwrap();
578        fs::write(blob2, b"blob2").unwrap();
579        fs::write(meta_package, b"meta_package").unwrap();
580        fs::write(meta_data, b"meta_data").unwrap();
581
582        let creation_manifest = PackageBuildManifest::from_dir(dir.path()).unwrap();
583        let far_contents = creation_manifest.far_contents();
584        let external_contents = creation_manifest.external_contents();
585        assert!(far_contents.contains_key("meta/data"));
586        assert!(far_contents.contains_key("meta/package"));
587        assert!(external_contents.contains_key("blob1"));
588        assert!(external_contents.contains_key("blob2"));
589    }
590
591    #[test]
592    fn test_from_pm_fini_not_found() {
593        let dir = tempfile::tempdir().unwrap();
594
595        let path = dir.path().join("path");
596        let not_found = dir.path().join("not_found");
597
598        fs::write(&path, b"hello world").unwrap();
599
600        let fini = format!(
601            "path={path}\n\
602             path={not_found}\n",
603            path = path.to_str().unwrap(),
604            not_found = not_found.to_str().unwrap()
605        );
606
607        assert_matches!(
608            PackageBuildManifest::from_pm_fini(fini.as_bytes()),
609            Err(PackageBuildManifestError::IoError(err)) if err.kind() == io::ErrorKind::NotFound
610        );
611    }
612
613    #[cfg(not(target_os = "fuchsia"))]
614    #[cfg(unix)]
615    #[test]
616    fn test_from_pm_fini_link() {
617        let dir = tempfile::tempdir().unwrap();
618
619        let path = dir.path().join("path");
620        let hard = dir.path().join("hard");
621        let sym = dir.path().join("symlink");
622
623        fs::write(&path, b"hello world").unwrap();
624        fs::hard_link(&path, &hard).unwrap();
625        std::os::unix::fs::symlink(&path, &sym).unwrap();
626
627        let fini = format!(
628            "path={path}\n\
629             path={hard}\n\
630             path={sym}\n",
631            path = path.to_str().unwrap(),
632            hard = hard.to_str().unwrap(),
633            sym = sym.to_str().unwrap(),
634        );
635
636        assert_eq!(
637            PackageBuildManifest::from_pm_fini(fini.as_bytes()).unwrap(),
638            PackageBuildManifest(VersionedPackageBuildManifest::Version1(PackageBuildManifestV1 {
639                external_contents: BTreeMap::from([(
640                    "path".to_string(),
641                    path.to_str().unwrap().to_string()
642                ),]),
643                far_contents: BTreeMap::new(),
644            })),
645        );
646    }
647
648    proptest! {
649        #[test]
650        fn test_from_external_and_far_contents_does_not_modify_valid_maps(
651            ref external_resource_path in random_external_resource_path(),
652            ref external_host_path in ".{0,30}",
653            ref far_resource_path in random_far_resource_path(),
654            ref far_host_path in ".{0,30}"
655        ) {
656            let external_contents = BTreeMap::from([
657                (external_resource_path.to_string(), external_host_path.to_string()),
658            ]);
659            let far_resource_path = format!("meta/{far_resource_path}");
660            let far_contents = BTreeMap::from([
661                (far_resource_path, far_host_path.to_string()),
662            ]);
663
664            let creation_manifest = PackageBuildManifest::from_external_and_far_contents(
665                external_contents.clone(), far_contents.clone())
666                .unwrap();
667
668            prop_assert_eq!(creation_manifest.external_contents(), &external_contents);
669            prop_assert_eq!(creation_manifest.far_contents(), &far_contents);
670        }
671    }
672}
fuchsia_pkg/package_build_manifest.rs

fuchsia_pkg/
package_build_manifest.rs
