We are hosting a .NET Core 3.1 application using Azure's Web App Service (Service plan: S1: 2) scaled out to 2 instances.
Using .NET data protection we get the certificates by thumbprint from Azure key vault.
The code snippet which adds data protection:
public static IServiceCollection AddCertDataProtection(this IServiceCollection services, SigningKeyCertificateOptions certOptions, ILogger log)
{
var primaryCert = Certificates.GetCertificateByThumbprint(certOptions.PrimaryThumbprint, log);
var secondaryCert = Certificates.GetCertificateByThumbprint(certOptions.SecondaryThumbprint, log);
if(primaryCert == null && secondaryCert == null)
{
throw new Exception($"Could not load primary or secondary certs. Primary: {certOptions.PrimaryThumbprint}," +
$" Secondary: {certOptions.SecondaryThumbprint}");
}
IDataProtectionBuilder dataProtectionBuilder = null;
if (primaryCert != null) {
dataProtectionBuilder = services.AddDataProtection()
.SetApplicationName("AppName")
.ProtectKeysWithCertificate(primaryCert);
}
if (secondaryCert != null)
{
dataProtectionBuilder?.UnprotectKeysWithAnyCertificate(secondaryCert);
}
dataProtectionBuilder.PersistKeysToDbContext<ApplicationDataProtectionContext>();
return services;
}
The certificates are found but we are getting the below exception when calling the PersistKeysToDbContext.
{"Message":"An exception occurred while processing the key element '\"<key id=\\\"XXXX\\\" version=\\\"1\\\" />\"'.","Element":"<key id=\"XXXXXX\" version=\"1\" />","EventId":{"Id":24,"Name":"ExceptionOccurredWhileProcessingTheKeyElement"},"SourceContext":"Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager","Application":"XXX","ApplicationName":"XXX","HostName":"XXX","Release":"XXXX","Level":"Error","Exception":{"Type":"Internal.Cryptography.CryptoThrowHelper+WindowsCryptographicException","Message":"Keyset does not exist","StackTrace":" at Internal.NativeCrypto.CapiHelper.CreateProvHandle(CspParameters parameters, Boolean randomKeyContainer)\r\n at System.Security.Cryptography.RSACryptoServiceProvider.get_SafeProvHandle()\r\n at System.Security.Cryptography.RSACryptoServiceProvider.get_SafeKeyHandle()\r\n at System.Security.Cryptography.RSACryptoServiceProvider..ctor(Int32 keySize, CspParameters parameters, Boolean useDefaultKeySize)\r\n at System.Security.Cryptography.RSACryptoServiceProvider..ctor(CspParameters parameters)\r\n at Internal.Cryptography.Pal.CertificatePal.<>c.<GetRSAPrivateKey>b__66_0(CspParameters csp)\r\n at Internal.Cryptography.Pal.CertificatePal.GetPrivateKey[T](Func`2 createCsp, Func`2 createCng)\r\n at Internal.Cryptography.Pal.CertificatePal.GetRSAPrivateKey()\r\n at Internal.Cryptography.Pal.CertificateExtensionsCommon.GetPrivateKey[T](X509Certificate2 certificate, Predicate`1 matchesConstraints)\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.EncryptedXmlDecryptor.EncryptedXmlWithCertificateKeys.GetKeyFromCert(EncryptedKey encryptedKey, KeyInfoX509Data keyInfo)\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.EncryptedXmlDecryptor.EncryptedXmlWithCertificateKeys.DecryptEncryptedKey(EncryptedKey encryptedKey)\r\n at System.Security.Cryptography.Xml.EncryptedXml.GetDecryptionKey(EncryptedData encryptedData, String symmetricAlgorithmUri)\r\n at System.Security.Cryptography.Xml.EncryptedXml.DecryptDocument()\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.EncryptedXmlDecryptor.Decrypt(XElement encryptedElement)\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.XmlEncryptionExtensions.DecryptElement(XElement element, IActivator activator)\r\n at Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager.Microsoft.AspNetCore.DataProtection.KeyManagement.Internal.IInternalXmlKeyManager.DeserializeDescriptorFromKeyElement(XElement keyElement)","InnerExceptions":[]},"timestamp":"2022-11-10T08:59:49.1113128+00:00"}
{"Message":"An exception occurred while processing the key element '\"<key id=\\\"XXX\\\" version=\\\"1\\\" />\"'.","Element":"<key id=\"XXX\" version=\"1\" />","EventId":{"Id":24,"Name":"ExceptionOccurredWhileProcessingTheKeyElement"},"SourceContext":"Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager","Application":"XXX","ApplicationName":"XXX","HostName":"XXX","Release":"XXX","Level":"Error","Exception":{"Type":"System.Security.Cryptography.CryptographicException","Message":"Unable to retrieve the decryption key.","StackTrace":" at System.Security.Cryptography.Xml.EncryptedXml.GetDecryptionKey(EncryptedData encryptedData, String symmetricAlgorithmUri)\r\n at System.Security.Cryptography.Xml.EncryptedXml.DecryptDocument()\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.EncryptedXmlDecryptor.Decrypt(XElement encryptedElement)\r\n at Microsoft.AspNetCore.DataProtection.XmlEncryption.XmlEncryptionExtensions.DecryptElement(XElement element, IActivator activator)\r\n at Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager.Microsoft.AspNetCore.DataProtection.KeyManagement.Internal.IInternalXmlKeyManager.DeserializeDescriptorFromKeyElement(XElement keyElement)","InnerExceptions":[]},"timestamp":"2022-11-10T08:59:48.7435261+00:00"}
I would like to mention that:
- The error only appears occasionally, sometimes it occurs only on one instance.
- We've added the app registration to the key vault's access policies
- We've set WEBSITE_LOAD_USER_PROFILE to *
- The certificates are loaded into the app's TLS/SSL -> Private keys correctly and it is not expired.
I am able to reproduce the same issue on local machine by removing the access rights of the current user with mmc.exe -> manage private keys for the certificate.
What configuration are we missing from the app service? Why does it only happen occasionally?
Any advice is more than welcome.
