Revert "Merged PR 685164: Forward rocksdb logs to Kusto"

This reverts commit 60bffd1.

Public/Src/Cache/ContentStore/Distributed/MetadataService/RocksDbContentMetadataDatabase.cs

@@ -4,6 +4,9 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.ContractsLight;
+using System.Diagnostics.SymbolStore;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -14,6 +17,7 @@
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Hashing;
+using BuildXL.Cache.ContentStore.Interfaces.Extensions;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Synchronization;
@@ -28,16 +32,18 @@
 using BuildXL.Native.IO;
 using BuildXL.Utilities;
 using BuildXL.Utilities.Collections;
+using BuildXL.Utilities.Serialization;
 using BuildXL.Utilities.Tasks;
 using RocksDbSharp;
-using static BuildXL.Cache.ContentStore.Distributed.MetadataService.RocksDbOperations;
 using static BuildXL.Engine.Cache.KeyValueStores.RocksDbStore;
 using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
 
 #nullable enable
 
 namespace BuildXL.Cache.ContentStore.Distributed.MetadataService
 {
+    using static RocksDbOperations;
+
     public class RocksDbContentMetadataDatabaseConfiguration : RocksDbContentLocationDatabaseConfiguration
     {
         public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
@@ -54,7 +60,7 @@ public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
     }
 
     /// <summary>
-    /// RocksDb-based version of <see cref="ContentLocationDatabase"/> used by the global location store.
+    /// RocksDb-based version of <see cref="ContentLocationDatabase"/>.
     /// </summary>
     public class RocksDbContentMetadataDatabase : ContentLocationDatabase
     {
@@ -283,27 +289,23 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
                 var dbAlreadyExists = Directory.Exists(storeLocation);
                 Directory.CreateDirectory(storeLocation);
 
-                var settings = new RocksDbStoreConfiguration(storeLocation)
-                               {
-                                   AdditionalColumns = ColumnNames.SelectMany(n => n),
-                                   RotateLogsMaxFileSizeBytes = 0L,
-                                   RotateLogsNumFiles = 60,
-                                   RotateLogsMaxAge = TimeSpan.FromHours(12),
-                                   EnableStatistics = true,
-                                   FastOpen = true,
-                                   LeveledCompactionDynamicLevelTargetSizes = true,
-                                   Compression = Compression.Zstd,
-                                   UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
-                                   UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
-                                   MergeOperators = GetMergeOperators()
-                               };
-
-                RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentMetadataDatabase));
-
-                Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
+                Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}'");
 
                 var possibleStore = KeyValueStoreAccessor.Open(
-                    settings,
+                    new RocksDbStoreConfiguration(storeLocation)
+                    {
+                        AdditionalColumns = ColumnNames.SelectMany(n => n),
+                        RotateLogsMaxFileSizeBytes = 0L,
+                        RotateLogsNumFiles = 60,
+                        RotateLogsMaxAge = TimeSpan.FromHours(12),
+                        EnableStatistics = true,
+                        FastOpen = true,
+                        LeveledCompactionDynamicLevelTargetSizes = true,
+                        Compression = Compression.Zstd,
+                        UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
+                        UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
+                        MergeOperators = GetMergeOperators()
+                    },
                     // When an exception is caught from within methods using the database, this handler is called to
                     // decide whether the exception should be rethrown in user code, and the database invalidated. Our
                     // policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.

Public/Src/Cache/ContentStore/Distributed/NuCache/ContentLocationDatabaseConfiguration.cs

@@ -5,7 +5,6 @@
 using BuildXL.Cache.ContentStore.Distributed.NuCache.InMemory;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.Host.Configuration;
-using RocksDbSharp;
 using static BuildXL.Utilities.ConfigurationHelper;
 
 #nullable enable
@@ -61,12 +60,6 @@ public abstract class ContentLocationDatabaseConfiguration
         /// </summary>
         public bool TraceOperations { get; set; } = true;
 
-        /// <summary>
-        /// Ges or sets log level from RocksDb emitted to Kusto.
-        /// Null - the tracing is off.
-        /// </summary>
-        public LogLevel? RocksDbTracingLevel { get; set; }
-
         /// <summary>
         /// Specifies whether to trace touches or not.
         /// Tracing touches is expensive in terms of the amount of traffic to Kusto and in terms of memory traffic.
@@ -223,7 +216,6 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
 
             configuration.RocksDbPerformanceSettings = settings.RocksDbPerformanceSettings;
 
-            ApplyIfNotNull(settings.ContentLocationDatabaseRocksDbTracingLevel, v => configuration.RocksDbTracingLevel = (LogLevel)v);
             ApplyIfNotNull(settings.TraceStateChangeDatabaseOperations, v => configuration.TraceOperations = v);
             ApplyIfNotNull(settings.TraceNoStateChangeDatabaseOperations, v => configuration.TraceNoStateChangeOperations = v);
 
@@ -232,7 +224,7 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
             ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionMaximumSizeMb, v => configuration.MetadataGarbageCollectionMaximumSizeMb = v);
             ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionLogEnabled, v => configuration.MetadataGarbageCollectionLogEnabled = v);
 
-            ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = (v && !settings.IsMasterEligible));
+            ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = v && !settings.IsMasterEligible);
             ApplyIfNotNull(settings.UseMergeOperatorForContentLocations, v => configuration.UseMergeOperatorForContentLocations = v);
 
             ApplyIfNotNull(settings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => configuration.LogsRetention = TimeSpan.FromMinutes(v));

Public/Src/Cache/ContentStore/Distributed/NuCache/RocksDbContentLocationDatabase.cs

@@ -9,7 +9,6 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Distributed.MetadataService;
 using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.Extensions;
@@ -27,6 +26,7 @@
 using BuildXL.Engine.Cache.KeyValueStores;
 using BuildXL.Native.IO;
 using BuildXL.Utilities;
+using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.ConfigurationHelpers;
 using BuildXL.Utilities.Serialization;
 using BuildXL.Utilities.Tasks;
@@ -213,7 +213,7 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
                 bool dbAlreadyExists = Directory.Exists(storeLocation);
                 Directory.CreateDirectory(storeLocation);
 
-                Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
+                Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}'");
 
                 var settings = new RocksDbStoreConfiguration(storeLocation)
                 {
@@ -243,8 +243,6 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
                     UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = _configuration.UseReadOptionsWithSetTotalOrderSeekInGarbageCollection,
                 };
 
-                RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentLocationDatabase));
-
                 if (_configuration.UseMergeOperatorForContentLocations)
                 {
                     var mergeContext = context.CreateNested(nameof(RocksDbContentLocationDatabase), caller: "MergeContentLocationEntries");

Public/Src/Cache/ContentStore/Distributed/Services/DistributedContentStoreServices.cs

@@ -218,7 +218,6 @@ private RocksDbContentMetadataStore CreateRocksDbContentMetadataStore()
                 MetadataSizeRotationThreshold = DistributedContentSettings.GlobalCacheMetadataSizeRotationThreshold,
             };
 
-            ApplyIfNotNull(DistributedContentSettings.ContentMetadataDatabaseRocksDbTracingLevel, v => dbConfig.RocksDbTracingLevel = (RocksDbSharp.LogLevel)v);
             ApplyIfNotNull(DistributedContentSettings.LocationEntryExpiryMinutes, v => dbConfig.ContentRotationInterval = TimeSpan.FromMinutes(v));
             dbConfig.MetadataRotationInterval = DistributedContentSettings.ContentMetadataServerMetadataRotationInterval;
 

Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs

@@ -220,21 +220,6 @@ public void DisableRedis()
         [DataMember]
         public bool? TraceStateChangeDatabaseOperations { get; set; }
 
-        /// <summary>
-        /// Defines a tracing level used by RocksDb logging adapter.
-        /// </summary>
-        /// <remarks>
-        /// See RocksDbSharp.LogLevel. The levels are:
-        /// 0 - Debug,
-        /// 1 - Info,
-        /// 2 - Warn,
-        /// 3 - Error,
-        /// 4 - Fatal,
-        /// 5 - Header
-        /// </remarks>
-        [DataMember]
-        public int? ContentLocationDatabaseRocksDbTracingLevel { get; set; } = 0;
-
         [DataMember]
         public bool TraceTouches { get; set; } = true;
 
@@ -961,10 +946,6 @@ public void DisableRedis()
         [DataMember]
         public bool ContentMetadataUseMergeWrites { get; set; }
 
-        /// <inheritdoc cref="ContentLocationDatabaseRocksDbTracingLevel"/>
-        [DataMember]
-        public int? ContentMetadataDatabaseRocksDbTracingLevel { get; set; } = 0;
-
         [DataMember]
         public string ContentMetadataBlobSecretName { get; set; }
 

Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStore.cs

@@ -113,8 +113,6 @@ private record struct ColumnFamilyInfo
 
         private readonly RocksDbStoreConfiguration? m_options;
 
-        private readonly RocksDbLoggingAdapter? m_loggingAdapter;
-
         /// <summary>
         /// Encapsulates <see cref="RocksDb"/> options that should be set.
         /// </summary>
@@ -156,12 +154,6 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
                 // }
                 .IncreaseParallelism(performanceConfiguration.GetBackgroundCompactionActualThreadCount());
 
-            if (configuration.HandleLogMessage != null)
-            {
-                m_loggingAdapter = new RocksDbLoggingAdapter(configuration.HandleLogMessage);
-                m_defaults.DbOptions = m_defaults.DbOptions.SetInfoLog(m_loggingAdapter);
-            }
-
             if (performanceConfiguration.DbWriteBufferSize is { } dbWriteBufferSize)
             {
                 m_defaults.DbOptions.SetDbWriteBufferSize(dbWriteBufferSize);
@@ -244,11 +236,6 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
                     .SetPrefixExtractor(SliceTransform.CreateNoOp())
                     .SetLevelCompactionDynamicLevelBytes(configuration.LeveledCompactionDynamicLevelTargetSizes);
 
-                if (configuration.HandleLogMessage != null)
-                {
-                    options = options.SetInfoLogLevel((int)configuration.LogLevel);
-                }
-
                 ColumnFamilyPerformanceConfiguration? perfConfiguration = null;
                 if (name != null)
                 {
@@ -965,9 +952,7 @@ public void Dispose()
                     }
                 }
 
-                // Disabling the log to avoid execution engine exceptions by calling deleted delegates
                 m_store.Dispose();
-                m_loggingAdapter?.Dispose();
             }
             else
             {

Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStoreConfiguration.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using BuildXL.Utilities.Collections;
 using RocksDbSharp;
 
 namespace BuildXL.Engine.Cache.KeyValueStores
@@ -160,14 +161,6 @@ public record RocksDbStoreConfiguration(string StoreDirectory)
         /// </remarks>
         public bool UseReadOptionsWithSetTotalOrderSeekInGarbageCollection { get; init; } = true;
 
-        /// <summary>
-        /// Log handler that will be called once rocksdb traces a message.
-        /// </summary>
-        public LogLineCallback? HandleLogMessage { get; set; }
-
-        /// <nodoc />
-        public LogLevel LogLevel { get; set; } = LogLevel.Info;
-
         /// <nodoc />
         public RocksDbPerformanceConfiguration PerformanceConfiguration { get; set; } = new RocksDbPerformanceConfiguration();
     }

cg/nuget/cgmanifest.json

@@ -2652,7 +2652,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "RocksDbNative",
-          "Version": "6.10.2-b20221102.4"
+          "Version": "6.10.2-b20220621.2"
         }
       }
     },
@@ -2661,7 +2661,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "RocksDbSharp",
-          "Version": "6.10.2-b20221102.4"
+          "Version": "6.10.2-b20220621.2"
         }
       }
     },

config.dsc

@@ -178,11 +178,11 @@ config({
                 { id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },
 
                 // RocksDb
-                { id: "RocksDbSharp", version: "6.10.2-b20221102.4", alias: "RocksDbSharpSigned", 
+                { id: "RocksDbSharp", version: "6.10.2-b20220621.2", alias: "RocksDbSharpSigned", 
                     dependentPackageIdsToSkip: [ "System.Memory" ],
                     dependentPackageIdsToIgnore: [ "System.Memory" ]
                 },
-                { id: "RocksDbNative", version: "6.10.2-b20221102.4" },
+                { id: "RocksDbNative", version: "6.10.2-b20220621.2" },
 
                 { id: "JsonDiffPatch.Net", version: "2.1.0" },