From 26e5eee055dc9bd49ff10bfbf8f95fb46698d5bc Mon Sep 17 00:00:00 2001 From: Marcin Sas-Szymanski Date: Sat, 17 Nov 2018 22:16:45 +0100 Subject: [PATCH 1/3] Handle serialization for RegistryPackage --- src/cargo/core/dependency.rs | 16 ++++++++ src/cargo/core/interning.rs | 14 ++++++- src/cargo/ops/registry.rs | 8 +--- src/cargo/sources/registry/mod.rs | 67 +++++++++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/cargo/core/dependency.rs b/src/cargo/core/dependency.rs index 262c3753874..2bccc235ad9 100644 --- a/src/cargo/core/dependency.rs +++ b/src/cargo/core/dependency.rs @@ -148,6 +148,22 @@ impl ser::Serialize for Kind { } } +impl fmt::Display for Kind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl Kind { + pub fn as_str(&self) -> &'static str { + match *self { + Kind::Normal => "normal", + Kind::Build => "build", + Kind::Development => "dev", + } + } +} + impl Dependency { /// Attempt to create a `Dependency` from an entry in the manifest. pub fn parse( diff --git a/src/cargo/core/interning.rs b/src/cargo/core/interning.rs index b5766e73790..38a90ad46ec 100644 --- a/src/cargo/core/interning.rs +++ b/src/cargo/core/interning.rs @@ -1,6 +1,6 @@ use serde::{Serialize, Serializer}; -use std::borrow::Borrow; +use std::borrow::{Borrow, Cow}; use std::cmp::Ordering; use std::collections::HashSet; use std::fmt; @@ -105,3 +105,15 @@ impl Serialize for InternedString { serializer.serialize_str(self.inner) } } + +impl From for Cow<'static, str> { + fn from(s: InternedString) -> Self { + s.as_str().into() + } +} + +impl<'a> From<&'a InternedString> for Cow<'static, str> { + fn from(s: &InternedString) -> Self { + s.as_str().into() + } +} diff --git a/src/cargo/ops/registry.rs b/src/cargo/ops/registry.rs index 9d159459b1e..4f2701cbe53 100644 --- a/src/cargo/ops/registry.rs +++ b/src/cargo/ops/registry.rs @@ -10,7 +10,6 @@ use curl::easy::{Easy, InfoType, SslOpt}; use log::{log, Level}; use url::percent_encoding::{percent_encode, QUERY_ENCODE_SET}; -use crate::core::dependency::Kind; use crate::core::manifest::ManifestMetadata; use crate::core::source::Source; use crate::core::{Package, SourceId, Workspace}; @@ -173,12 +172,7 @@ fn transmit( features: dep.features().iter().map(|s| s.to_string()).collect(), version_req: dep.version_req().to_string(), target: dep.platform().map(|s| s.to_string()), - kind: match dep.kind() { - Kind::Normal => "normal", - Kind::Build => "build", - Kind::Development => "dev", - } - .to_string(), + kind: dep.kind().to_string(), registry: dep_registry, explicit_name_in_toml: dep.explicit_name_in_toml().map(|s| s.to_string()), }) diff --git a/src/cargo/sources/registry/mod.rs b/src/cargo/sources/registry/mod.rs index 2c618c904f0..e985703bb6e 100644 --- a/src/cargo/sources/registry/mod.rs +++ b/src/cargo/sources/registry/mod.rs @@ -161,12 +161,13 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::fs::File; +use std::io::Read; use std::path::{Path, PathBuf}; use flate2::read::GzDecoder; use log::debug; use semver::Version; -use serde::Deserialize; +use serde::{Serialize, Deserialize}; use tar::Archive; use crate::core::dependency::{Dependency, Kind}; @@ -176,7 +177,7 @@ use crate::sources::PathSource; use crate::util::errors::CargoResultExt; use crate::util::hex; use crate::util::to_url::ToUrl; -use crate::util::{internal, CargoResult, Config, FileLock, Filesystem}; +use crate::util::{internal, CargoResult, Config, FileLock, Filesystem, Sha256}; const INDEX_LOCK: &str = ".cargo-index-lock"; pub const CRATES_IO_INDEX: &str = "https://github.com/rust-lang/crates.io-index"; @@ -214,7 +215,7 @@ pub struct RegistryConfig { pub api: Option, } -#[derive(Deserialize)] +#[derive(Serialize, Deserialize)] pub struct RegistryPackage<'a> { name: Cow<'a, str>, vers: Version, @@ -222,6 +223,7 @@ pub struct RegistryPackage<'a> { features: BTreeMap, Vec>>, cksum: String, yanked: Option, + #[serde(skip_serializing_if = "Option::is_none")] links: Option>, } @@ -270,7 +272,7 @@ enum Field { Links, } -#[derive(Deserialize)] +#[derive(Serialize, Deserialize)] struct RegistryDependency<'a> { name: Cow<'a, str>, req: Cow<'a, str>, @@ -279,10 +281,48 @@ struct RegistryDependency<'a> { default_features: bool, target: Option>, kind: Option>, + #[serde(skip_serializing_if = "Option::is_none")] registry: Option>, + #[serde(skip_serializing_if = "Option::is_none")] package: Option>, } +impl<'a> RegistryPackage<'a> { + pub fn from_package(pkg: &Package, mut pkg_file: FileLock) -> CargoResult { + let cksum = { + let mut c = Vec::new(); + pkg_file.read_to_end(&mut c)?; + + let mut sha = Sha256::new(); + sha.update(&c); + ::hex::encode(&sha.finish()) + }; + let summary = pkg.summary(); + + Ok( + Self { + name: pkg.name().as_str().into(), + vers: pkg.version().clone(), + deps: pkg.dependencies().into_iter().map(RegistryDependency::from_dep).collect(), + features: summary + .features() + .into_iter() + .map(|(name, values)| ( + name.as_str().into(), + values + .into_iter() + .map(|value| value.to_string(summary).into()) + .collect()) + ) + .collect(), + cksum, + yanked: Some(false), + links: pkg.summary().links().map(|link| link.as_str().into()) + } + ) + } +} + impl<'a> RegistryDependency<'a> { /// Converts an encoded dependency in the registry to a cargo dependency pub fn into_dep(self, default: SourceId) -> CargoResult { @@ -335,6 +375,25 @@ impl<'a> RegistryDependency<'a> { Ok(dep) } + + pub fn from_dep(dep: &Dependency) -> Self { + let (name, package) = match dep.explicit_name_in_toml() { + Some(explicit) => (explicit.into(), Some(dep.package_name().into())), + None => (dep.package_name().into(), None), + }; + + Self { + name, + req: dep.version_req().to_string().into(), + features: dep.features().into_iter().map(|f| f.into()).collect(), + optional: dep.is_optional(), + default_features: dep.uses_default_features(), + target: dep.platform().map(|s| s.to_string().into()), + kind: Some(dep.kind().as_str().into()), + registry: dep.registry_id().map(|s| s.url().to_string().into()), + package + } + } } pub trait RegistryData { From 0494587b36830546a68eea799b5d41eaa6676c6d Mon Sep 17 00:00:00 2001 From: Marcin Sas-Szymanski Date: Sat, 17 Nov 2018 22:19:14 +0100 Subject: [PATCH 2/3] Add tests for generate-index-metadata --- tests/testsuite/generate_index_metadata.rs | 462 +++++++++++++++++++++ tests/testsuite/main.rs | 1 + 2 files changed, 463 insertions(+) create mode 100644 tests/testsuite/generate_index_metadata.rs diff --git a/tests/testsuite/generate_index_metadata.rs b/tests/testsuite/generate_index_metadata.rs new file mode 100644 index 00000000000..3c9cd466a6c --- /dev/null +++ b/tests/testsuite/generate_index_metadata.rs @@ -0,0 +1,462 @@ +use crate::support::project; +use crate::support::registry::{self, Package}; + +#[test] +fn missing_package() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + p.cargo("generate-index-metadata") + .with_status(101) + .with_stderr(&format!( + "\ +error: could not find package file. Ensure that crate has been packaged using `cargo package` + +Caused by: + failed to open: {path} + +Caused by: + [..] +", + path = p + .root() + .join("target") + .join("package") + .join("foo-0.0.1.crate") + .display() + )) + .run(); +} + +#[test] +fn simple_project() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [], + "features": {}, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} + +#[test] +fn project_with_deps() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [dependencies] + bar = "0.0.4" + baz = "*" + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("baz", "0.0.2").publish(); + Package::new("nested", "0.0.3").publish(); + Package::new("bar", "0.0.4").dep("nested", "*").publish(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [ + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "bar", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "^0.0.4", + "target": null + }, + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "baz", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + } + ], + "features": {}, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} + +#[test] +fn project_with_dev_and_build_deps() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [dev-dependencies] + bar = "*" + + [build-dependencies] + baz= { version = "*", optional = true } + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("baz", "0.0.1").publish(); + Package::new("bar", "0.0.1").publish(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [ + { + "default_features": true, + "features": [], + "kind": "dev", + "name": "bar", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + }, + { + "default_features": true, + "features": [], + "kind": "build", + "name": "baz", + "optional": true, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + } + ], + "features": {}, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} + +#[test] +fn project_with_deps_from_alternative_registry() { + let p = project() + .file( + "Cargo.toml", + r#" + cargo-features = ["alternative-registries"] + + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [dependencies] + bar = "*" + baz = { version = "*", registry = "alternative" } + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("baz", "0.0.1").alternative(true).publish(); + Package::new("bar", "0.0.1").publish(); + + p.cargo("package").masquerade_as_nightly_cargo().run(); + + p.cargo("generate-index-metadata") + .masquerade_as_nightly_cargo() + .with_json(&format!( + r#" + {{ + "name": "foo", + "vers": "0.0.1", + "deps": [ + {{ + "default_features": true, + "features": [], + "kind": "normal", + "name": "bar", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + }}, + {{ + "default_features": true, + "features": [], + "kind": "normal", + "name": "baz", + "optional": false, + "registry": "{reg}", + "req": "*", + "target": null + }} + ], + "features": {{}}, + "cksum": "[..]", + "yanked": false + }}"#, + reg = registry::alt_registry() + )) + .run(); +} + +#[test] +fn project_with_features() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [features] + default = ["jquery", "session"] + go-faster = [] + secure-password = ["bcrypt"] + session = ["cookie/session"] + + [dependencies] + cookie = "*" + jquery = { version = "*", optional = true } + bcrypt = { version = "*", optional = true } + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("cookie", "0.0.1") + .feature("session", &[]) + .publish(); + Package::new("jquery", "0.0.1").publish(); + Package::new("bcrypt", "0.0.1").publish(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [ + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "bcrypt", + "optional": true, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + }, + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "cookie", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + }, + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "jquery", + "optional": true, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + } + ], + "features": { + "default": [ + "jquery", + "session" + ], + "go-faster": [], + "secure-password": [ + "bcrypt" + ], + "session": [ + "cookie/session" + ] + }, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} + +#[test] +fn project_with_dep_with_features() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [dependencies] + bar = { version = "*", default-features = false, features = ["baz"] } + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("bar", "0.0.1").feature("baz", &[]).publish(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [ + { + "default_features": false, + "features": ["baz"], + "kind": "normal", + "name": "bar", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": null + } + ], + "features": {}, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} + +#[test] +fn project_with_platform_specific_dep() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = 'foo' + documentation = 'foo' + + [target.'cfg(windows)'.dependencies] + winapi = "*" + "#, + ) + .file("src/main.rs", "fn main() {}") + .build(); + + Package::new("winapi", "0.0.1").publish(); + + p.cargo("package").run(); + + p.cargo("generate-index-metadata") + .with_json( + r#" + { + "name": "foo", + "vers": "0.0.1", + "deps": [ + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "winapi", + "optional": false, + "registry": "https://github.com/rust-lang/crates.io-index", + "req": "*", + "target": "cfg(windows)" + } + ], + "features": {}, + "cksum": "[..]", + "yanked": false + }"#, + ) + .run(); +} diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs index 6100f5d6b2b..49e6eee993d 100644 --- a/tests/testsuite/main.rs +++ b/tests/testsuite/main.rs @@ -38,6 +38,7 @@ mod features; mod fetch; mod fix; mod freshness; +mod generate_index_metadata; mod generate_lockfile; mod git; mod init; From 440939ee3ed9bc19dc90f2e426bbc678a69ce86e Mon Sep 17 00:00:00 2001 From: Marcin Sas-Szymanski Date: Sat, 17 Nov 2018 22:18:04 +0100 Subject: [PATCH 3/3] Add generate-index-metadata subcommand --- .../cargo/commands/generate_index_metadata.rs | 22 +++++++++++++++++++ src/bin/cargo/commands/mod.rs | 3 +++ .../ops/cargo_generate_index_metadata.rs | 18 +++++++++++++++ src/cargo/ops/mod.rs | 2 ++ 4 files changed, 45 insertions(+) create mode 100644 src/bin/cargo/commands/generate_index_metadata.rs create mode 100644 src/cargo/ops/cargo_generate_index_metadata.rs diff --git a/src/bin/cargo/commands/generate_index_metadata.rs b/src/bin/cargo/commands/generate_index_metadata.rs new file mode 100644 index 00000000000..18125124820 --- /dev/null +++ b/src/bin/cargo/commands/generate_index_metadata.rs @@ -0,0 +1,22 @@ +use cargo::ops; +use cargo::print_json; +use crate::command_prelude::*; + +pub fn cli() -> App { + subcommand("generate-index-metadata") + .about( + "Output index file metadata, \ + required by registry", + ) + .arg_manifest_path() +} + +pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { + let ws = args.workspace(config)?; + + let result = ops::generate_index_metadata(&ws)?; + + print_json(&result); + + Ok(()) +} diff --git a/src/bin/cargo/commands/mod.rs b/src/bin/cargo/commands/mod.rs index c62f2aad211..42858d68ad6 100644 --- a/src/bin/cargo/commands/mod.rs +++ b/src/bin/cargo/commands/mod.rs @@ -9,6 +9,7 @@ pub fn builtin() -> Vec { doc::cli(), fetch::cli(), fix::cli(), + generate_index_metadata::cli(), generate_lockfile::cli(), git_checkout::cli(), init::cli(), @@ -44,6 +45,7 @@ pub fn builtin_exec(cmd: &str) -> Option) -> Cli "doc" => doc::exec, "fetch" => fetch::exec, "fix" => fix::exec, + "generate-index-metadata" => generate_index_metadata::exec, "generate-lockfile" => generate_lockfile::exec, "git-checkout" => git_checkout::exec, "init" => init::exec, @@ -79,6 +81,7 @@ pub mod clean; pub mod doc; pub mod fetch; pub mod fix; +pub mod generate_index_metadata; pub mod generate_lockfile; pub mod git_checkout; pub mod init; diff --git a/src/cargo/ops/cargo_generate_index_metadata.rs b/src/cargo/ops/cargo_generate_index_metadata.rs new file mode 100644 index 00000000000..be5fb7d357b --- /dev/null +++ b/src/cargo/ops/cargo_generate_index_metadata.rs @@ -0,0 +1,18 @@ +use crate::core::Workspace; +use crate::sources::registry::RegistryPackage; +use crate::util::errors::{CargoResult, CargoResultExt}; + +/// Generate index metadata for packages +pub fn generate_index_metadata<'a>(ws: &Workspace<'_>) -> CargoResult> { + let pkg = ws.current()?; + let config = ws.config(); + let filename = format!("{}-{}.crate", pkg.name(), pkg.version()); + let dir = ws.target_dir().join("package"); + + dir.open_ro(filename, config, "package file") + .and_then(|pkg_file| RegistryPackage::from_package(pkg, pkg_file)) + .chain_err(|| { + "could not find package file. Ensure that crate has been packaged using `cargo package`" + }) + .map_err(Into::into) +} diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index b8a3104f193..75a23be67b3 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -3,6 +3,7 @@ pub use self::cargo_compile::{compile, compile_with_exec, compile_ws, CompileOpt pub use self::cargo_compile::{CompileFilter, FilterRule, Packages}; pub use self::cargo_doc::{doc, DocOptions}; pub use self::cargo_fetch::{fetch, FetchOptions}; +pub use self::cargo_generate_index_metadata::generate_index_metadata; pub use self::cargo_generate_lockfile::generate_lockfile; pub use self::cargo_generate_lockfile::update_lockfile; pub use self::cargo_generate_lockfile::UpdateOptions; @@ -30,6 +31,7 @@ mod cargo_clean; mod cargo_compile; mod cargo_doc; mod cargo_fetch; +mod cargo_generate_index_metadata; mod cargo_generate_lockfile; mod cargo_install; mod cargo_new;