Update Azure Storage SDK to v12, add logging for Azure Blob storage directory

build/build.xml

@@ -1,4 +1,4 @@
 <build>
-  <project id="Examine" version="1.0.6"/>
-  <project id="Examine.AzureDirectory" version="1.0.0-beta07"/> 
-</build>
+  <project id="Examine" version="1.0.7"/>
+  <project id="Examine.AzureDirectory" version="1.0.0-beta08"/> 
+</build>

src/Examine.AzureDirectory/AzureDirectoryFactory.cs

@@ -1,9 +1,14 @@
 using System.Configuration;
 using System.IO;
+using Examine.LuceneEngine.DeletePolicies;
 using Examine.LuceneEngine.Directories;
-using Examine.LuceneEngine.Providers;
+using Examine.LuceneEngine.MergePolicies;
+using Examine.LuceneEngine.MergeShedulers;
+using Examine.Providers;
+using Lucene.Net.Analysis;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
-using Microsoft.Azure.Storage;
+using static Lucene.Net.Index.IndexWriter;
 
 namespace Examine.AzureDirectory
 {
@@ -13,9 +18,9 @@ namespace Examine.AzureDirectory
     public class AzureDirectoryFactory : SyncTempEnvDirectoryFactory, IDirectoryFactory
     {
         private readonly bool _isReadOnly;
-
         public AzureDirectoryFactory()
         {
+            _isReadOnly = false;
         }
 
         public AzureDirectoryFactory(bool isReadOnly)
@@ -47,19 +52,61 @@ public AzureDirectoryFactory(bool isReadOnly)
         /// </returns>
         public override Lucene.Net.Store.Directory CreateDirectory(DirectoryInfo luceneIndexFolder)
         {
-            var indexFolder = luceneIndexFolder;
-            var tempFolder = GetLocalStorageDirectory(indexFolder);
-
-            return new AzureDirectory(
-                CloudStorageAccount.Parse(ConfigurationManager.AppSettings[ConfigStorageKey]),
-                ConfigurationManager.AppSettings[ConfigContainerKey],
-                new SimpleFSDirectory(tempFolder),
+            var directory = new AzureLuceneDirectory(
+                GetStorageAccountConnectionString(),
+                GetContainerName(),
+                GetLocalCacheDirectory(luceneIndexFolder),
                 rootFolder: luceneIndexFolder.Name,
-                isReadOnly: _isReadOnly);
+                isReadOnly: GetIsReadOnly());
+
+            directory.IsReadOnly = _isReadOnly;
+            directory.SetMergePolicyAction(e => new NoMergePolicy(e));
+            directory.SetMergeScheduler(new NoMergeSheduler());
+            directory.SetDeletion(new NoDeletionPolicy());
+            return directory;
         }
 
+
         // Explicit implementation, see https://github.com/Shazwazza/Examine/pull/153
         Lucene.Net.Store.Directory IDirectoryFactory.CreateDirectory(DirectoryInfo luceneIndexFolder) => CreateDirectory(luceneIndexFolder);
 
+        /// <summary>
+        /// Gets the Local Cache Lucence Directory
+        /// </summary>
+        /// <param name="luceneIndexFolder">The lucene index folder.</param>
+        /// <returns>The <see cref="Lucene.Net.Store.Directory"/> used by Lucence as the local cache directory</returns>
+        protected virtual Lucene.Net.Store.Directory GetLocalCacheDirectory(DirectoryInfo luceneIndexFolder)
+        {
+            return new SimpleFSDirectory(luceneIndexFolder);
+        }
+
+        /// <summary>
+        /// Gets the Cloud Storage Account
+        /// </summary>
+        /// <remarks>Retrieves connection string from <see cref="ConfigStorageKey"/></remarks>
+        /// <returns>CloudStorageAccount</returns>
+        protected virtual string GetStorageAccountConnectionString()
+        {
+            return ConfigurationManager.AppSettings[ConfigStorageKey];
+        }
+
+        /// <summary>
+        /// Retrieve the container name
+        /// </summary>
+        /// <remarks>Retrieves the container name from <see cref="ConfigContainerKey"/></remarks>
+        /// <returns>Name of the container where the indexes are stored</returns>
+        protected virtual string GetContainerName()
+        {
+            return ConfigurationManager.AppSettings[ConfigContainerKey];
+        }
+
+        /// <summary>
+        /// Get whether the index is readonly
+        /// </summary>
+        /// <returns></returns>
+        protected virtual bool GetIsReadOnly()
+        {
+            return _isReadOnly;
+        }
     }
 }

src/Examine.AzureDirectory/AzureDirectorySimpleLockFactory.cs

@@ -8,9 +8,9 @@ namespace Examine.AzureDirectory
     /// </summary>
     public class AzureDirectorySimpleLockFactory : LockFactory
     {
-        private readonly AzureDirectory _azureDirectory;
+        private readonly AzureLuceneDirectory _azureDirectory;
 
-        public AzureDirectorySimpleLockFactory(AzureDirectory azureDirectory)
+        public AzureDirectorySimpleLockFactory(AzureLuceneDirectory azureDirectory)
         {
             _azureDirectory = azureDirectory;
         }
@@ -30,10 +30,13 @@ public override void ClearLock(string name)
 
             var lockFile = _azureDirectory.RootFolder + name;
 
-            var blob = _azureDirectory.BlobContainer.GetBlockBlobReference(lockFile);
-            var flag1 = blob.Exists();
+            var blob = _azureDirectory.BlobContainer.GetBlobClient(lockFile);
+            var flag1Response = blob.Exists();
+            var flag1 = flag1Response.Value;
             bool flag2;
-            if (blob.Exists())
+
+            var flag2Response = blob.Exists();
+            if (flag2Response.Value)
             {
                 blob.Delete();
                 flag2 = true;

src/Examine.AzureDirectory/AzureIndexInput.cs

@@ -2,10 +2,11 @@
 using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Threading;
+using Azure.Storage.Blobs;
 using Examine.LuceneEngine.Directories;
 using Lucene.Net.Store;
-using Microsoft.Azure.Storage.Blob;
 
 namespace Examine.AzureDirectory
 {
@@ -14,19 +15,19 @@ namespace Examine.AzureDirectory
     /// </summary>
     public class AzureIndexInput : IndexInput
     {
-        private AzureDirectory _azureDirectory;
-        private CloudBlobContainer _blobContainer;
-        private ICloudBlob _blob;
+        private AzureLuceneDirectory _azureDirectory;
+        private BlobClient _blob;
         private readonly string _name;
 
         private IndexInput _indexInput;
         private readonly Mutex _fileMutex;
 
         public Lucene.Net.Store.Directory CacheDirectory => _azureDirectory.CacheDirectory;
 
-        public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
+        public AzureIndexInput(AzureLuceneDirectory azuredirectory, BlobClient blob)
         {
             _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
+            _name = _name.Split(new string[] { "%2F" }, StringSplitOptions.RemoveEmptyEntries).Last();
             _azureDirectory = azuredirectory ?? throw new ArgumentNullException(nameof(azuredirectory));
 #if FULLDEBUG
             Trace.WriteLine($"opening {_name} ");
@@ -35,7 +36,6 @@ public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
             _fileMutex.WaitOne();
             try
             {                
-                _blobContainer = azuredirectory.BlobContainer;
                 _blob = blob;
 
                 var fileName = _name;
@@ -48,12 +48,15 @@ public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
                 else
                 {
                     var cachedLength = CacheDirectory.FileLength(fileName);
-                    var hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out var blobLengthMetadata); 
-                    var blobLength = blob.Properties.Length;
+
+                    var blobPropertiesResponse = blob.GetProperties();
+                    var blobProperties = blobPropertiesResponse.Value;
+                    var hasMetadataValue = blobProperties.Metadata.TryGetValue("CachedLength", out var blobLengthMetadata); 
+                    var blobLength = blobProperties.ContentLength;
                     if (hasMetadataValue) long.TryParse(blobLengthMetadata, out blobLength);
 
-                    var blobLastModifiedUtc = blob.Properties.LastModified.Value.UtcDateTime;
-                    if (blob.Metadata.TryGetValue("CachedLastModified", out var blobLastModifiedMetadata))
+                    var blobLastModifiedUtc = blobProperties.LastModified.UtcDateTime;
+                    if (blobProperties.Metadata.TryGetValue("CachedLastModified", out var blobLastModifiedMetadata))
                     {
                         if (long.TryParse(blobLastModifiedMetadata, out var longLastModified))
                             blobLastModifiedUtc = new DateTime(longLastModified).ToUniversalTime();
@@ -98,7 +101,7 @@ public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
                         using (var fileStream = new StreamOutput(CacheDirectory.CreateOutput(fileName)))
                         {
                             // get the blob
-                            _blob.DownloadToStream(fileStream);
+                            _blob.DownloadTo(fileStream);
 
                             fileStream.Flush();
 #if FULLDEBUG
@@ -133,7 +136,7 @@ private void InflateStream(string fileName)
             using (var deflatedStream = new MemoryStream())
             {
                 // get the deflated blob
-                _blob.DownloadToStream(deflatedStream);
+                _blob.DownloadTo(deflatedStream);
 
 #if FULLDEBUG
                 Trace.WriteLine($"GET {_name} RETREIVED {deflatedStream.Length} bytes");
@@ -162,12 +165,10 @@ public AzureIndexInput(AzureIndexInput cloneInput)
         {
             _name = cloneInput._name;
             _azureDirectory = cloneInput._azureDirectory;
-            _blobContainer = cloneInput._blobContainer;
             _blob = cloneInput._blob;
 
             if (string.IsNullOrWhiteSpace(_name)) throw new ArgumentNullException(nameof(cloneInput._name));
             if (_azureDirectory == null) throw new ArgumentNullException(nameof(cloneInput._azureDirectory));
-            if (_blobContainer == null) throw new ArgumentNullException(nameof(cloneInput._blobContainer));
             if (_blob == null) throw new ArgumentNullException(nameof(cloneInput._blob));
 
             _fileMutex = SyncMutexManager.GrabMutex(cloneInput._azureDirectory, cloneInput._name);
@@ -220,7 +221,6 @@ protected override void Dispose(bool isDiposing)
                 _indexInput.Dispose();
                 _indexInput = null;
                 _azureDirectory = null;
-                _blobContainer = null;
                 _blob = null;
                 GC.SuppressFinalize(this);
             }

src/Examine.AzureDirectory/AzureIndexInputFactory.cs

@@ -0,0 +1,18 @@
+using Azure.Storage.Blobs;
+using Lucene.Net.Store;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Examine.AzureDirectory
+{
+    public class AzureIndexInputFactory : IAzureIndexInputFactory
+    {
+        public IndexInput GetIndexInput(AzureLuceneDirectory azuredirectory, BlobClient blob)
+        {
+            return new AzureIndexInput(azuredirectory, blob);
+        }
+    }
+}

src/Examine.AzureDirectory/AzureIndexOutput.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
 using System.Threading;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Examine.LuceneEngine.Directories;
 using Lucene.Net.Store;
-using Microsoft.Azure.Storage.Blob;
 
 namespace Examine.AzureDirectory
 {
@@ -15,15 +17,16 @@ namespace Examine.AzureDirectory
     /// </summary>
     public class AzureIndexOutput : IndexOutput
     {
-        private readonly AzureDirectory _azureDirectory;
+        private readonly AzureLuceneDirectory _azureDirectory;
         //private CloudBlobContainer _blobContainer;
         private readonly string _name;
         private IndexOutput _indexOutput;
         private readonly Mutex _fileMutex;
-        private ICloudBlob _blob;
+        private BlobClient _blob;
+
         public Lucene.Net.Store.Directory CacheDirectory => _azureDirectory.CacheDirectory;
 
-        public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob, string name)
+        public AzureIndexOutput(AzureLuceneDirectory azureDirectory, BlobClient blob, string name)
         {
             //NOTE: _name was null here, is this intended? https://github.com/azure-contrib/AzureDirectory/issues/19 I have changed this to be correct now
             _name = name;
@@ -32,9 +35,7 @@ public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob, string n
             _fileMutex.WaitOne();
             try
             {                
-                //_blobContainer = _azureDirectory.BlobContainer;
                 _blob = blob;
-                _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
 
                 // create the local cache one we will operate against...
                 _indexOutput = CacheDirectory.CreateOutput(_name);
@@ -91,16 +92,23 @@ protected override void Dispose(bool isDisposing)
                     try
                     {
                         // push the blobStream up to the cloud
-                        _blob.UploadFromStream(blobStream);
+                        _blob.Upload(blobStream,overwrite:true);
 
                         // set the metadata with the original index file properties
-                        _blob.Metadata["CachedLength"] = originalLength.ToString();
-                        _blob.Metadata["CachedLastModified"] = CacheDirectory.FileModified(fileName).ToString();
-                        _blob.SetMetadata();
+                        var metadata = new Dictionary<string, string>();
+                        metadata.Add("CachedLength", originalLength.ToString());
+                        metadata.Add("CachedLastModified", CacheDirectory.FileModified(fileName).ToString());
+
+                        var response = _blob.SetMetadata(metadata);
 #if FULLDEBUG
                         Trace.WriteLine($"PUT {blobStream.Length} bytes to {_name} in cloud");
 #endif
                     }
+                    catch(Azure.RequestFailedException ex) when (ex.Status == 409)
+                    {
+                        //File already exists
+                        throw;
+                    }
                     finally
                     {
                         blobStream.Dispose();
@@ -123,7 +131,7 @@ protected override void Dispose(bool isDisposing)
             }
         }
 
-        private MemoryStream CompressStream(string fileName, long originalLength)
+        protected virtual MemoryStream CompressStream(string fileName, long originalLength)
         {
             // unfortunately, deflate stream doesn't allow seek, and we need a seekable stream
             // to pass to the blob storage stuff, so we compress into a memory stream

src/Examine.AzureDirectory/AzureIndexOutputFactory.cs

@@ -0,0 +1,18 @@
+using Azure.Storage.Blobs;
+using Lucene.Net.Store;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Examine.AzureDirectory
+{
+    public class AzureIndexOutputFactory : IAzureIndexOutputFactory
+    {
+        public IndexOutput CreateIndexOutput(AzureLuceneDirectory azureDirectory, BlobClient blob, string name)
+        {
+            return new AzureIndexOutput(azureDirectory, blob, name);
+        }
+    }
+}