From a1dc2e85b3ca22e50edcde0ee7878fef0e142d45 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Fri, 5 Apr 2024 16:46:32 +0000 Subject: [PATCH] [WIP] Realign outputs to be more extensible in the future Signed-off-by: Michael Lieberman --- Cargo.lock | 212 +++++++++++++++++++++++++---- skootrs-bin/src/helpers.rs | 27 +++- skootrs-lib/Cargo.toml | 1 + skootrs-lib/src/service/mod.rs | 6 +- skootrs-lib/src/service/output.rs | 128 +++++++++++++++++ skootrs-lib/src/service/project.rs | 9 +- skootrs-model/src/skootrs/mod.rs | 68 ++++++++- 7 files changed, 404 insertions(+), 47 deletions(-) create mode 100644 skootrs-lib/src/service/output.rs diff --git a/Cargo.lock b/Cargo.lock index f4e4717..31adeb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,8 +39,8 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2", - "http", + "h2 0.3.22", + "http 0.2.11", "httparse", "httpdate", "itoa", @@ -75,7 +75,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", - "http", + "http 0.2.11", "regex", "serde", "tracing", @@ -1662,7 +1662,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap 2.2.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap 2.2.3", "slab", "tokio", @@ -1758,6 +1777,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -1765,7 +1795,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1812,9 +1865,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -1826,6 +1879,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1833,8 +1906,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "log", "rustls", "rustls-native-certs", @@ -1848,7 +1921,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.27", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1861,12 +1934,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.27", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -2514,9 +2623,9 @@ dependencies = [ "either", "futures", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", "hyper-timeout", "jsonwebtoken", @@ -2551,9 +2660,9 @@ dependencies = [ "either", "futures", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", "hyper-timeout", "jsonwebtoken", @@ -3242,12 +3351,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -3278,6 +3387,48 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b48d98d932f4ee75e541614d32a7f44c889b72bd9c2e04d95edd135989df88" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "retain_mut" version = "0.1.7" @@ -3961,7 +4112,7 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "opentelemetry_sdk", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "serde_yaml", @@ -3988,6 +4139,7 @@ dependencies = [ "futures", "octocrab 0.33.3", "regress", + "reqwest 0.12.0", "schemars", "serde", "serde_json", @@ -4060,9 +4212,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "smol_str" @@ -4281,7 +4433,7 @@ dependencies = [ "radix_trie", "rand 0.8.5", "regex", - "reqwest", + "reqwest 0.11.24", "revision", "roaring", "rocksdb", @@ -4691,8 +4843,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "http-range-header", "iri-string", "pin-project-lite", @@ -4862,7 +5014,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 0.2.11", "httparse", "log", "rand 0.8.5", diff --git a/skootrs-bin/src/helpers.rs b/skootrs-bin/src/helpers.rs index 43a552d..f53da89 100644 --- a/skootrs-bin/src/helpers.rs +++ b/skootrs-bin/src/helpers.rs @@ -12,8 +12,8 @@ use skootrs_model::{ security_insights::insights10::SecurityInsightsVersion100YamlSchema, skootrs::{ Config, EcosystemInitializeParams, FacetGetParams, FacetMapKey, GithubRepoParams, - GithubUser, GoParams, InitializedProject, MavenParams, ProjectCreateParams, - ProjectGetParams, ProjectOutputParams, ProjectOutputType, RepoCreateParams, SkootError, + GithubUser, GoParams, InitializedProject, MavenParams, OutputGetParams, OutputType, + ProjectCreateParams, ProjectGetParams, RepoCreateParams, SkootError, SourceInitializeParams, SupportedEcosystems, SUPPORTED_ECOSYSTEMS, }, }; @@ -246,7 +246,7 @@ impl Output { pub async fn get<'a, T: ProjectService + ?Sized>( config: &Config, _project_service: &'a T, - project_output_params: Option, + project_output_params: Option, ) -> Result<(), SkootError> { let project_output_params = match project_output_params { Some(p) => p, @@ -263,7 +263,20 @@ impl Output { Ok(()) } - async fn prompt_project_output(_config: &Config) -> Result { + /// Prints out the list of project outputs. This includes things like SBOMs or SLSA attestations. + /// + /// # Errors + /// + /// Returns an error if the list of project outputs can't be fetched for some reason. + pub async fn print_list<'a, T: ProjectService + ?Sized>( + _config: &Config, + _project_service: &'a T, + project_get_params: Option, + ) -> Result<(), SkootError> { + unimplemented!() + } + + async fn prompt_project_output(_config: &Config) -> Result { let projects = Project::list().await?; let selected_project = inquire::Select::new("Select a project", projects.iter().collect()).prompt()?; @@ -320,11 +333,11 @@ impl Output { }; let selected_output_type_enum = match selected_output_type { - "SBOM" => ProjectOutputType::SBOM, - _ => ProjectOutputType::Custom("Other".to_string()), + "SBOM" => OutputType::SBOM, + _ => OutputType::Custom("Other".to_string()), }; - Ok(ProjectOutputParams { + Ok(OutputGetParams { project_url: selected_project.clone(), project_output_type: selected_output_type_enum, project_output: selected_output.clone(), diff --git a/skootrs-lib/Cargo.toml b/skootrs-lib/Cargo.toml index 99ac056..06ff6df 100644 --- a/skootrs-lib/Cargo.toml +++ b/skootrs-lib/Cargo.toml @@ -22,6 +22,7 @@ ahash = "0.8.7" sha2 = "0.10.8" url = "2.5.0" base64 = "0.21.7" +reqwest = "0.12.0" [dev-dependencies] tempdir = "0.3.7" diff --git a/skootrs-lib/src/service/mod.rs b/skootrs-lib/src/service/mod.rs index 9622ab4..9315937 100644 --- a/skootrs-lib/src/service/mod.rs +++ b/skootrs-lib/src/service/mod.rs @@ -1,4 +1,3 @@ - // // Copyright 2024 The Skootrs Authors. // @@ -14,8 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod ecosystem; +pub mod facet; +pub mod output; pub mod project; pub mod repo; pub mod source; -pub mod ecosystem; -pub mod facet; \ No newline at end of file diff --git a/skootrs-lib/src/service/output.rs b/skootrs-lib/src/service/output.rs new file mode 100644 index 0000000..324763a --- /dev/null +++ b/skootrs-lib/src/service/output.rs @@ -0,0 +1,128 @@ +// +// Copyright 2024 The Skootrs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(clippy::module_name_repetitions)] + +use octocrab::models::repos::{Asset, Release}; +use skootrs_model::skootrs::{Output, OutputDescriptor, OutputGetParams, OutputType, SkootError}; + +pub trait OutputService { + /// Gets an output for a Skootrs project. This is usually something related to a release like + /// an SBOM or SLSA attestation. + /// + /// # Errors + /// + /// Returns an error if the output doesn't exist or can't be fetched for some reason. + fn get( + &self, + params: OutputGetParams, + ) -> impl std::future::Future> + Send; +} + +pub struct LocalOutputService; + +impl OutputService for LocalOutputService { + async fn get(&self, params: OutputGetParams) -> Result {} +} + +pub struct GithubReleaseHandler; +impl GithubReleaseHandler { + pub async fn get_release( + owner: &str, + repo: &str, + release: Option<&str>, + ) -> Result { + let release = { + if let Some(release) = release { + octocrab::instance() + .repos(owner, repo) + .releases() + .get_by_tag(release) + .await? + } else { + octocrab::instance() + .repos(owner, repo) + .releases() + .get_latest() + .await? + } + }; + Ok(release) + } + + pub async fn list_outputs( + owner: &str, + repo: &str, + release: Option<&str>, + ) -> Result, SkootError> { + //let release = release.unwrap_or("latest"); + let release = GithubReleaseHandler::get_release(owner, repo, release).await?; + let descriptors = { + let assets = release.assets; + assets + .into_iter() + .map(|a| { + let output_type = GithubReleaseHandler::asset_to_output_type(&a); + OutputDescriptor { + name: a.name, + url: a.browser_download_url, + output_type, + } + }) + .collect() + }; + + Ok(descriptors) + } + + pub async fn get_output( + owner: &str, + repo: &str, + release: Option<&str>, + output_name: &str, + ) -> Result { + let release = GithubReleaseHandler::get_release(owner, repo, release).await?; + let asset = release + .assets + .into_iter() + .find(|a| a.name == output_name) + .ok_or("Output not found")?; + + let output_content = reqwest::get(asset.browser_download_url) + .await? + .text() + .await?; + + let output_type = GithubReleaseHandler::asset_to_output_type(&asset); + match output_type { + OutputType::SBOM => { + let sbom = skootrs_model::sbom::SBOM::from_str(&output_content)?; + Ok(Output::SBOM(sbom)) + } + OutputType::Custom(_) => Ok(Output::Custom(output_content)), + } + + Ok(()) + } + pub fn asset_to_output_type(asset: &Asset) -> OutputType { + let output_type: OutputType = match asset.url { + // Follows: https://github.com/ossf/sbom-everywhere/blob/main/reference/sbom_naming.md + _ if asset.name.contains(".spdx.") => OutputType::SBOM, + _ if asset.name.contains(".cdx.") => OutputType::SBOM, + _ => OutputType::Custom("Unknown".to_string()), + }; + output_type + } +} diff --git a/skootrs-lib/src/service/project.rs b/skootrs-lib/src/service/project.rs index b72cccf..87f6efd 100644 --- a/skootrs-lib/src/service/project.rs +++ b/skootrs-lib/src/service/project.rs @@ -25,7 +25,9 @@ use skootrs_model::skootrs::{ ProjectGetParams, SkootError, }; -use super::{ecosystem::EcosystemService, repo::RepoService, source::SourceService}; +use super::{ + ecosystem::EcosystemService, output::OutputService, repo::RepoService, source::SourceService, +}; use tracing::{debug, error, info}; /// The `ProjectService` trait provides an interface for initializing and managing a Skootrs project. @@ -79,19 +81,22 @@ pub struct LocalProjectService< ES: EcosystemService, SS: SourceService, FS: RootFacetService, + OS: OutputService, > { pub repo_service: RS, pub ecosystem_service: ES, pub source_service: SS, pub facet_service: FS, + pub output_service: OS, } -impl ProjectService for LocalProjectService +impl ProjectService for LocalProjectService where RS: RepoService + Send + Sync, ES: EcosystemService + Send + Sync, SS: SourceService + Send + Sync, FS: RootFacetService + Send + Sync, + OS: OutputService + Send + Sync, { async fn initialize( &self, diff --git a/skootrs-model/src/skootrs/mod.rs b/skootrs-model/src/skootrs/mod.rs index 8127306..f310c20 100644 --- a/skootrs-model/src/skootrs/mod.rs +++ b/skootrs-model/src/skootrs/mod.rs @@ -19,7 +19,7 @@ use std::{collections::HashMap, error::Error, fmt, str::FromStr}; use serde::{Deserialize, Serialize}; use strum::{EnumString, VariantNames}; -use url::Host; +use url::{Host, Url}; use utoipa::ToSchema; use self::facet::{InitializedFacet, SupportedFacetType}; @@ -153,11 +153,11 @@ pub struct ProjectGetParams { /// The paramaters for getting the output of a project, e.g. an SBOM from a release #[derive(Serialize, Deserialize, Clone, Debug)] #[cfg_attr(feature = "openapi", derive(ToSchema))] -pub struct ProjectOutputParams { +pub struct OutputGetParams { /// The URL of the Skootrs project to get the output from. pub project_url: String, /// The type of output to get from the project. - pub project_output_type: ProjectOutputType, + pub project_output_type: OutputType, // TODO: Should project_output be a part of the ProjectOutputType enum? /// The output to get from the project. pub project_output: String, @@ -166,13 +166,71 @@ pub struct ProjectOutputParams { /// The set of supported output types #[derive(Serialize, Deserialize, Clone, Debug)] #[cfg_attr(feature = "openapi", derive(ToSchema))] -pub enum ProjectOutputType { +pub enum OutputType { /// An output type for getting an SBOM from a project. - SBOM, + SBOM(OutputSBOMFormatType), /// An output type for getting a custom output from a project. Custom(String), } +/// The set of supported output types +#[derive(Serialize, Deserialize, Clone, Debug)] +pub enum Output { + /// An output type for getting an SBOM from a project. + SBOM(OutputSBOMFormat), + /// An output type for getting a custom output from a project. + Custom(String), +} + +/// The metadata for downloading an output from a project. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutputDescriptor { + /// The name of the output. Useful for identifying the output wihtout the full url. + pub name: String, + /// The URL to download the output. + pub url: Url, + /// The type of output this descriptor describes. + pub output_type: OutputType, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub enum OutputSBOMFormat { + SPDX(OutputSBOMSPDX), + CycloneDX(OutputSBOMCycloneDX), + Unknown(OutputSBOMUnknown), +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub enum OutputSBOMFormatType { + SPDX, + CycloneDX, + Unknown, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutputSBOMSPDX { + pub content: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutputSBOMCycloneDX {} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutputSBOMUnknown {} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub enum OutputLocation { + Github(GithubOutputLocation), +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GithubOutputLocation { + pub owner: String, + pub repo: String, + pub release: String, + pub output: String, +} + /// The parameters for getting a facet from a project. #[derive(Serialize, Deserialize, Clone, Debug)] #[cfg_attr(feature = "openapi", derive(ToSchema))]