Skip to content

Commit

Permalink
Merged PR 735951: Update SBOM related packages
Browse files Browse the repository at this point in the history
Updates various SBOM related packages to work with the new Component Detector.

These changes are necessary in order for CloudBuild Repo to update to the new version of Component Detector and SBOM tooling
  • Loading branch information
mpysson committed Aug 30, 2023
1 parent ddc826c commit 2af16fd
Show file tree
Hide file tree
Showing 8 changed files with 40 additions and 39 deletions.
18 changes: 9 additions & 9 deletions Public/Src/Tools/DropDaemon/DropDaemon.cs
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,7 @@ private async Task<IIpcResult> GenerateAndUploadBuildManifestFileWithSignedCatal
hasMoreData = bxlResult.Result.HasMoreData;
} while (hasMoreData);

IEnumerable<SBOMFile> manifestFileList = fileList.Select(ToSbomFile);
IEnumerable<SbomFile> manifestFileList = fileList.Select(ToSbomFile);

string sbomGenerationRootDirectory = null;
var logger = GetDropSpecificLogger(dropConfig);
Expand All @@ -905,13 +905,13 @@ private async Task<IIpcResult> GenerateAndUploadBuildManifestFileWithSignedCatal
FileUtilities.CreateDirectory(sbomGenerationRootDirectory);

// Always generate SPDX, but exclude CloudBuild manifest if configured to do so
var specs = new List<SBOMSpecification>() { new("SPDX", "2.2") };
var specs = new List<SbomSpecification>() { new("SPDX", "2.2") };
if (!m_disableCloudBuildManifest)
{
specs.Add(new("CloudBuildManifest", "1.0.0"));
}

Possible<IEnumerable<SBOMPackage>> maybePackages;
Possible<IEnumerable<SbomPackage>> maybePackages;
using (m_counters.StartStopwatch(DropDaemonCounter.BuildManifestComponentConversionDuration))
{
maybePackages = await GetSbomPackagesAsync(logger);
Expand All @@ -924,7 +924,7 @@ private async Task<IIpcResult> GenerateAndUploadBuildManifestFileWithSignedCatal

var packages = maybePackages.Result;
logger.Verbose("Starting SBOM Generation");
var result = await m_sbomGenerator.GenerateSBOMAsync(sbomGenerationRootDirectory, manifestFileList, packages, metadata, specs);
var result = await m_sbomGenerator.GenerateSbomAsync(sbomGenerationRootDirectory, manifestFileList, packages, metadata, specs);
logger.Verbose("Finished SBOM Generation");

if (!result.IsSuccessful)
Expand Down Expand Up @@ -991,7 +991,7 @@ private static string GetSbomGenerationErrorDetails(IList<EntityError> errors)
return sb.ToString();
}

private SBOMFile ToSbomFile(BuildXL.Ipc.ExternalApi.Commands.BuildManifestFileInfo fileInfo)
private SbomFile ToSbomFile(BuildXL.Ipc.ExternalApi.Commands.BuildManifestFileInfo fileInfo)
{
// Include artifacts hash only when computing CloudBuildV1 Manifest
var maybeArtifactsHash = m_disableCloudBuildManifest ? Array.Empty<ContentHash>() : new[] { fileInfo.AzureArtifactsHash };
Expand Down Expand Up @@ -1021,14 +1021,14 @@ static AlgorithmName mapHashType(HashType hashType)
}

/// <summary>
/// Tries to convert output from component detection to a list of <see cref="SBOMPackage"/>.
/// Tries to convert output from component detection to a list of <see cref="SbomPackage"/>.
/// </summary>
/// <returns>
/// A converted list of <see cref="SBOMPackage"/> if successful.
/// A converted list of <see cref="SbomPackage"/> if successful.
/// If not successful, errors messages will be logged and also returned as a Failure{string}.
/// Any warnings raised by the tooling are also logged via the APIServer.
/// </returns>
private async Task<Possible<IEnumerable<SBOMPackage>>> GetSbomPackagesAsync(IIpcLogger logger)
private async Task<Possible<IEnumerable<SbomPackage>>> GetSbomPackagesAsync(IIpcLogger logger)
{
// Read Path for bcde output from environment, this should already be set by Cloudbuild
var bcdeOutputJsonPath = Environment.GetEnvironmentVariable(Constants.ComponentGovernanceBCDEOutputFilePath);
Expand All @@ -1039,7 +1039,7 @@ private async Task<Possible<IEnumerable<SBOMPackage>>> GetSbomPackagesAsync(IIpc
// and the SBOM creation here can still happen without a set of packages.
// Log a message on the ApiServer it and return an empty set.
Analysis.IgnoreResult(await ApiClient.LogMessage($"[GetSbomPackages] The '{Constants.ComponentGovernanceBCDEOutputFilePath}' environment variable was not found. This happens when component governance on the build runner is disabled. Component detection data will not be included in build manifest.", isWarning: false));
return new List<SBOMPackage>();
return new List<SbomPackage>();
}
else if (!System.IO.File.Exists(bcdeOutputJsonPath))
{
Expand Down
14 changes: 7 additions & 7 deletions Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ export namespace DropDaemon {
appConfig: f`DropDaemon.exe.config`,
assemblyBindingRedirects: dropDaemonBindingRedirects(),
sources: globR(d`.`, "*.cs"),

references: [
importFrom("BuildXL.Cache.ContentStore").Hashing.dll,
importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll,
Expand Down Expand Up @@ -56,6 +55,7 @@ export namespace DropDaemon {
internalsVisibleTo: [
"Test.Tool.DropDaemon",
],
deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Text.Encodings.Web.dll`, a`System.Text.Json.dll`] },
runtimeContentToSkip: dropDaemonRuntimeContentToSkip()
});

Expand Down Expand Up @@ -136,15 +136,15 @@ export namespace DropDaemon {
name: "System.Text.Json",
publicKeyToken: "cc7b13ffcd2ddd51",
culture: "neutral",
oldVersion: "0.0.0.0-5.0.0.0",
newVersion: "5.0.0.0",
oldVersion: "0.0.0.0-7.0.0.0",
newVersion: "7.0.0.0"
},
{
name: "System.Text.Encodings.Web",
publicKeyToken: "cc7b13ffcd2ddd51",
culture: "neutral",
oldVersion: "0.0.0.0-5.0.0.1",
newVersion: "5.0.0.1", // Corresponds to { id: "System.Text.Encodings.Web", version: "4.7.2" },
oldVersion: "0.0.0.0-7.0.0.0",
newVersion: "7.0.0.0"
}
];
}
Expand Down Expand Up @@ -172,9 +172,9 @@ export namespace DropDaemon {
importFrom("Microsoft.ComponentDetection.Contracts").pkg,
importFrom("Microsoft.Sbom.Adapters").pkg,
importFrom("packageurl-dotnet").pkg,
importFrom("System.Text.Json.v5.0.0").pkg,
importFrom("System.Text.Json.v7.0.0").pkg,
importFrom("Newtonsoft.Json").pkg,
importFrom("System.Text.Encodings.Web.v5.0.1").pkg,
importFrom("System.Text.Encodings.Web.v7.0.0").pkg,
];
}

Expand Down
8 changes: 4 additions & 4 deletions Public/Src/Tools/UnitTests/DropDaemon/SbomGenerationTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,10 @@ public async Task GenerateSbom()

var sbomGenerationRootDirectory = Path.Combine(Path.GetTempPath(), "sbom");

var specs = new List<SBOMSpecification>() { new("SPDX", "2.2"), new("CloudBuildManifest", "1.0.0") };
var specs = new List<SbomSpecification>() { new("SPDX", "2.2"), new("CloudBuildManifest", "1.0.0") };

// A file with VSO and SHA1 hashes to generate both SPDX and CBManifest
var myfile = new SBOMFile()
var myfile = new SbomFile()
{
Id = "MyFileId",
Path = "Oh/What/A/Cool/Path.txt",
Expand All @@ -93,7 +93,7 @@ public async Task GenerateSbom()
},
}
};
IEnumerable<SBOMFile> files = new List<SBOMFile>() { myfile };
IEnumerable<SbomFile> files = new List<SbomFile>() { myfile };

var (adapterReport, packages) = new ComponentDetectionToSBOMPackageAdapter().TryConvert(GenerateBcdeOutput(Path.GetTempFileName()));
XAssert.IsNotNull(packages);
Expand All @@ -105,7 +105,7 @@ public async Task GenerateSbom()
}
}

var result = await sbomGenerator.GenerateSBOMAsync(sbomGenerationRootDirectory, files, packages, metadata, specs);
var result = await sbomGenerator.GenerateSbomAsync(sbomGenerationRootDirectory, files, packages, metadata, specs);
if (!result.IsSuccessful)
{
var errorDetails = GetSbomGenerationErrorDetails(result.Errors);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ namespace Test.Tool.DropDaemon {
...importFrom("BuildXL.Tools.DropDaemon").dropDaemonSbomPackages(),
],

deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Text.Encodings.Web.dll`, a`System.Text.Json.dll`] },
runtimeContentToSkip: importFrom("BuildXL.Tools.DropDaemon").dropDaemonRuntimeContentToSkip(),
});
}
18 changes: 9 additions & 9 deletions cg/nuget/cgmanifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.ComponentDetection.Contracts",
"Version": "2.0.2"
"Version": "3.6.4"
}
}
},
Expand Down Expand Up @@ -1779,7 +1779,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.Parsers.ManifestGenerator",
"Version": "2.3.3"
"Version": "3.7.0"
}
}
},
Expand Down Expand Up @@ -1815,7 +1815,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.SBOMCore",
"Version": "2.3.3"
"Version": "3.8.0"
}
}
},
Expand All @@ -1824,7 +1824,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.Sbom.Adapters",
"Version": "0.2.6"
"Version": "1.5.2"
}
}
},
Expand All @@ -1833,7 +1833,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.Sbom.Contracts",
"Version": "0.2.6"
"Version": "1.5.2"
}
}
},
Expand All @@ -1842,7 +1842,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.Sbom.Extensions",
"Version": "0.2.6"
"Version": "1.5.2"
}
}
},
Expand All @@ -1851,7 +1851,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.Sbom.Parsers.Spdx22SbomParser",
"Version": "0.2.6"
"Version": "1.5.2"
}
}
},
Expand Down Expand Up @@ -4029,7 +4029,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "System.Text.Encodings.Web",
"Version": "5.0.1"
"Version": "7.0.0"
}
}
},
Expand All @@ -4047,7 +4047,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "System.Text.Json",
"Version": "5.0.0"
"Version": "7.0.0"
}
}
},
Expand Down
2 changes: 1 addition & 1 deletion config.dsc
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ config({

// Needed for SBOM Generation
{ id: "Microsoft.Extensions.Logging.Abstractions", version: "6.0.3", alias: "Microsoft.Extensions.Logging.Abstractions.v6.0.3", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"] },
{ id: "System.Text.Encodings.Web", version: "5.0.1", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"], alias: "System.Text.Encodings.Web.v5.0.1" },
{ id: "System.Text.Encodings.Web", version: "7.0.0", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"], alias: "System.Text.Encodings.Web.v7.0.0" },
{ id: "packageurl-dotnet", version: "1.1.0" },
{ id: "System.Reactive", version: "4.4.1" },

Expand Down
14 changes: 7 additions & 7 deletions config.microsoftInternal.dsc
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,13 @@ export const pkgs = isMicrosoftInternal ? [
{ id: "VisualCppTools.Internal.VS2017Layout", version: "14.16.27034", osSkip: [ "macOS", "unix" ] },

// SBOM Generation
{ id: "Microsoft.SBOMCore", version: "2.3.3" },
{ id: "Microsoft.Parsers.ManifestGenerator", version: "2.3.3", dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
{ id: "Microsoft.Sbom.Parsers.Spdx22SbomParser", version: "0.2.6" },
{ id: "Microsoft.Sbom.Adapters", version: "0.2.6" },
{ id: "Microsoft.ComponentDetection.Contracts", version: "2.0.2" },
{ id: "Microsoft.Sbom.Contracts", version: "0.2.6" },
{ id: "Microsoft.Sbom.Extensions", version: "0.2.6", dependentPackageIdsToSkip: ["System.Text.Json"] },
{ id: "Microsoft.SBOMCore", version: "3.8.0" },
{ id: "Microsoft.Parsers.ManifestGenerator", version: "3.7.0", dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] },
{ id: "Microsoft.Sbom.Parsers.Spdx22SbomParser", version: "1.5.2" },
{ id: "Microsoft.Sbom.Adapters", version: "1.5.2" },
{ id: "Microsoft.ComponentDetection.Contracts", version: "3.6.4" },
{ id: "Microsoft.Sbom.Contracts", version: "1.5.2" },
{ id: "Microsoft.Sbom.Extensions", version: "1.5.2", dependentPackageIdsToSkip: ["System.Text.Json"] },
{ id: "Microsoft.Bcl.HashCode", version: "1.1.1" },

// Process remoting
Expand Down
4 changes: 2 additions & 2 deletions config.nuget.dotnetcore.dsc
Original file line number Diff line number Diff line change
Expand Up @@ -244,9 +244,9 @@ export const pkgs = [
{ id: "System.Text.Json", version: "4.7.2",
dependentPackageIdsToSkip: ["System.Memory", "System.ValueTuple", "System.Runtime.CompilerServices.Unsafe", "System.Numerics.Vectors", "System.Threading.Tasks.Extensions", "Microsoft.Bcl.AsyncInterfaces"],
},
{ id: "System.Text.Json", version: pkgVersion5,
{ id: "System.Text.Json", version: pkgVersion7,
dependentPackageIdsToSkip: ["System.Memory", "System.Buffers", "System.ValueTuple", "System.Runtime.CompilerServices.Unsafe", "System.Numerics.Vectors", "System.Threading.Tasks.Extensions", "Microsoft.Bcl.AsyncInterfaces", "System.Text.Encodings.Web"],
alias: "System.Text.Json.v5.0.0"
alias: "System.Text.Json.v7.0.0"
},
{ id: "System.Threading.AccessControl", version: pkgVersionNext },

Expand Down

0 comments on commit 2af16fd

Please sign in to comment.