Update Azure Storage SDK to v12, add logging for Azure Blob storage directory

src/Examine.AzureDirectory/AzureDirectoryFactory.cs

@@ -62,7 +62,7 @@ public override Lucene.Net.Store.Directory CreateDirectory(DirectoryInfo luceneI
             directory.IsReadOnly = _isReadOnly;
             directory.SetMergePolicyAction(e => new NoMergePolicy(e));
             directory.SetMergeScheduler(new NoMergeSheduler());
-            directory.SetDeletion(NoDeletionPolicy.INSTANCE);
+            directory.SetDeletion(new NoDeletionPolicy());
             return directory;
         }
 

src/Examine.AzureDirectory/AzureIndexInput.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Threading;
 using Azure.Storage.Blobs;
 using Examine.LuceneEngine.Directories;
@@ -26,6 +27,7 @@ public class AzureIndexInput : IndexInput
         public AzureIndexInput(AzureLuceneDirectory azuredirectory, BlobClient blob)
         {
             _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
+            _name = _name.Split(new string[] { "%2F" }, StringSplitOptions.RemoveEmptyEntries).Last();
             _azureDirectory = azuredirectory ?? throw new ArgumentNullException(nameof(azuredirectory));
 #if FULLDEBUG
             Trace.WriteLine($"opening {_name} ");

src/Examine.AzureDirectory/AzureLuceneDirectory.cs

@@ -107,7 +107,6 @@ protected virtual BlobContainerClient GetBlobContainerClient(string containerNam
 
         public string RootFolder { get; }
         public bool CompressBlobs { get; }
-        public Lucene.Net.Store.Directory CacheDirectory { get; protected set; }
 
         public void ClearCache()
         {
@@ -118,7 +117,7 @@ public void ClearCache()
                 CacheDirectory.DeleteFile(file);
             }
         }
-        public void RebuildCache()
+        public virtual void RebuildCache()
         {
             Trace.WriteLine($"INFO Rebuilding index cache {RootFolder}");
             try
@@ -160,7 +159,7 @@ internal string[] GetAllBlobFiles()
             {
                 return results.ToArray();
             }
-            var names = results.Select(x => x.Replace(RootFolder, "")).ToArray();
+            var names = results.Where(x => !x.EndsWith(".lock")).Select(x => x.Replace(RootFolder, "")).ToArray();
             return names;
         }
 

src/Examine.AzureDirectory/AzureReadOnlyLuceneDirectory.cs

@@ -66,47 +66,54 @@ private void CreateOrReadCache()
 
             }
         }
-
-        public void RebuildCache()
+        private object _rebuildLock = new object();
+        public override void RebuildCache()
         {
-
-            Trace.WriteLine("INFO Rebuilding cache");
-            var tempDir = new DirectoryInfo(
-                Path.Combine(_cacheDirectoryPath,
-                    _cacheDirectoryName, DateTimeOffset.UtcNow.ToString("yyyyMMddTHHmmssfffffff")));
-            if (tempDir.Exists == false)
-                tempDir.Create();
-            Lucene.Net.Store.Directory newIndex = new SimpleFSDirectory(tempDir);
-            foreach (string file in GetAllBlobFiles())
+            lock (_rebuildLock)
             {
-             //   newIndex.TouchFile(file);
-                var blob = _blobContainer.GetBlobClient(RootFolder + file);
-                SyncFile(newIndex,blob, file);
-            }
 
-            var oldIndex = CacheDirectory;
-            newIndex.Dispose();
-            newIndex = new SimpleFSDirectory(tempDir);
-
-            CacheDirectory = newIndex;
-            _lockFactory = newIndex.LockFactory;
-            if (oldIndex != null)
-            {
-                oldIndex.ClearLock("write.lock");
-                foreach (var file in oldIndex.ListAll())
+                //Needs locking
+                Trace.WriteLine("INFO Rebuilding cache");
+                var tempDir = new DirectoryInfo(
+                    Path.Combine(_cacheDirectoryPath,
+                        _cacheDirectoryName, DateTimeOffset.UtcNow.ToString("yyyyMMddTHHmmssfffffff")));
+                if (tempDir.Exists == false)
+                    tempDir.Create();
+                Lucene.Net.Store.Directory newIndex = new SimpleFSDirectory(tempDir);
+                foreach (string file in GetAllBlobFiles())
                 {
-                    if (oldIndex.FileExists(file))
+                    //   newIndex.TouchFile(file);
+                    if ("write.lock".Equals(file))
                     {
-                        oldIndex.DeleteFile(file);
+                        continue;
                     }
+                    var blob = _blobContainer.GetBlobClient(RootFolder + file);
+                    SyncFile(newIndex, blob, file);
                 }
-                oldIndex.Dispose();
-               DirectoryInfo oldindex  = new DirectoryInfo(Path.Combine(_cacheDirectoryPath,
-                   _cacheDirectoryName,OldIndexFolderName));
-               oldindex.Delete();
+
+                var oldIndex = CacheDirectory;
+                newIndex.Dispose();
+                newIndex = new SimpleFSDirectory(tempDir);
+
+                CacheDirectory = newIndex;
+                _lockFactory = newIndex.LockFactory;
+                if (oldIndex != null)
+                {
+                    oldIndex.ClearLock("write.lock");
+                    foreach (var file in oldIndex.ListAll())
+                    {
+                        if (oldIndex.FileExists(file))
+                        {
+                            oldIndex.DeleteFile(file);
+                        }
+                    }
+                    oldIndex.Dispose();
+                    DirectoryInfo oldindex = new DirectoryInfo(Path.Combine(_cacheDirectoryPath,
+                        _cacheDirectoryName, OldIndexFolderName));
+                    oldindex.Delete();
+                }
+                OldIndexFolderName = tempDir.Name;
             }
-            OldIndexFolderName = tempDir.Name;
-
         }
         private void SyncFile(Directory newIndex, BlobClient blob, string fileName)
         {
@@ -150,52 +157,12 @@ protected void InflateStream(Lucene.Net.Store.Directory newIndex, BlobClient blo
 
             }
         }
-        public override string[] ListAll()
-        {
-            return CacheDirectory.ListAll();
-        }
-
-        public override bool FileExists(string name)
-        {
-            return CacheDirectory.FileExists(name);
-        }
-
-        public override long FileModified(string name)
-        {
-            return CacheDirectory.FileModified(name);
-        }
-
-        public override void TouchFile(string name)
-        {
-             CacheDirectory.TouchFile(name);
-        }
-
-        public override void DeleteFile(string name)
-        {
-            CacheDirectory.DeleteFile(name);
-        }
-
-        public override long FileLength(string name)
-        {
-            return CacheDirectory.FileLength(name);
-        }
-
-        public override IndexOutput CreateOutput(string name)
-        {
-            return CacheDirectory.CreateOutput(name);
-        }
-
-        public override IndexInput OpenInput(string name)
-        {
-            CheckDirty();
-            return CacheDirectory.OpenInput(name);
-        }
-
-        protected override void Dispose(bool disposing)
-        {
-            CacheDirectory.Dispose();
-        }
-        protected virtual void HandleOutOfSync()
+
+        //protected override void Dispose(bool disposing)
+        //{
+        //    CacheDirectory.Dispose();
+        //}
+        protected override void HandleOutOfSync()
         {
             RebuildCache();
         }

src/Examine.AzureDirectory/ReadOnlyAzureDirectoryFactory.cs

@@ -31,7 +31,7 @@ public override Lucene.Net.Store.Directory CreateDirectory(DirectoryInfo luceneI
             directory.IsReadOnly = _isReadOnly;
             directory.SetMergePolicyAction(e => new NoMergePolicy(e));
             directory.SetMergeScheduler(new NoMergeSheduler());
-            directory.SetDeletion(NoDeletionPolicy.INSTANCE);
+            directory.SetDeletion(new NoDeletionPolicy());
             return directory;
         }
         protected string GetLocalStorageDirectory(DirectoryInfo indexPath)

src/Examine.Test/AzureDirectoryTests/IntegrationTests.cs

@@ -36,14 +36,13 @@ public void TestReadAndWrite()
 
                 azureDirectory.SetMergePolicyAction(e => new NoMergePolicy(e));
                 azureDirectory.SetMergeScheduler(new NoMergeSheduler());
-                azureDirectory.SetDeletion(NoDeletionPolicy.INSTANCE);
+                azureDirectory.SetDeletion(new NoDeletionPolicy());
                 using (var indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), !IndexReader.IndexExists(azureDirectory),
                     azureDirectory.GetDeletionPolicy(), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
                 {
                     indexWriter.SetRAMBufferSizeMB(10.0); 
                     indexWriter.SetMergePolicy(azureDirectory.GetMergePolicy(indexWriter));
                     indexWriter.SetMergeScheduler(azureDirectory.GetMergeScheduler());
-
                     for (int iDoc = 0; iDoc < 10000; iDoc++)
                     {
                         var doc = new Document();
@@ -92,7 +91,7 @@ public void TestReadOnlyAndWrite()
 
                 azureReadWriteDirectory.SetMergePolicyAction(e => new NoMergePolicy(e));
                 azureReadWriteDirectory.SetMergeScheduler(new NoMergeSheduler());
-                azureReadWriteDirectory.SetDeletion(NoDeletionPolicy.INSTANCE);
+                azureReadWriteDirectory.SetDeletion(new NoDeletionPolicy());
                 using (var indexWriter = new IndexWriter(azureReadWriteDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),
                     !IndexReader.IndexExists(azureReadWriteDirectory), azureReadWriteDirectory.GetDeletionPolicy(),
                     new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
@@ -127,7 +126,7 @@ public void TestReadOnlyAndWrite()
                 var azureReadOnlyDirectory = new AzureReadOnlyLuceneDirectory(connectionString, containerName, readonlyDirectoryFolder.FullName, containerName);
                 azureReadOnlyDirectory.SetMergePolicyAction(e => new NoMergePolicy(e));
                 azureReadOnlyDirectory.SetMergeScheduler(new NoMergeSheduler());
-                azureReadOnlyDirectory.SetDeletion(NoDeletionPolicy.INSTANCE);
+                azureReadOnlyDirectory.SetDeletion(new NoDeletionPolicy());
                 for (var i = 0; i < 100; i++)
                 {
                     using (var searcher = new IndexSearcher(azureReadOnlyDirectory))
@@ -156,6 +155,100 @@ public void TestReadOnlyAndWrite()
 
         }
 
+        [Explicit("Requires storage emulator to be running")]
+        [Test]
+        public void TestWritingToReadonly()
+        {
+            var connectionString = Environment.GetEnvironmentVariable("DataConnectionString") ?? "UseDevelopmentStorage=true";
+            string containerName = "testcatalog";
+
+            var readWriteCacheDirectory = new RandomIdRAMDirectory();
+            AzureLuceneDirectory azureReadWriteDirectory = null;
+            AzureReadOnlyLuceneDirectory azureReadOnlyDirectory = null;
+            var readonlyDirectoryFolder = new DirectoryInfo(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "TEMP", containerName));
+            try
+            {
+                // default AzureDirectory stores cache in local temp folder
+                azureReadWriteDirectory = new AzureLuceneDirectory(connectionString, containerName, readWriteCacheDirectory);
+
+                azureReadWriteDirectory.SetMergePolicyAction(e => new NoMergePolicy(e));
+                azureReadWriteDirectory.SetMergeScheduler(new NoMergeSheduler());
+                azureReadWriteDirectory.SetDeletion(new NoDeletionPolicy());
+                using (var indexWriter = new IndexWriter(azureReadWriteDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),
+                    !IndexReader.IndexExists(azureReadWriteDirectory), azureReadWriteDirectory.GetDeletionPolicy(),
+                    new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
+                {
+
+                    indexWriter.SetRAMBufferSizeMB(10.0);
+                    indexWriter.SetMergePolicy(azureReadWriteDirectory.GetMergePolicy(indexWriter));
+                    indexWriter.SetMergeScheduler(azureReadWriteDirectory.GetMergeScheduler());
+
+                    for (int iDoc = 0; iDoc < 10000; iDoc++)
+                    {
+                        var doc = new Document();
+                        doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString() + "-" + iDoc.ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
+                        doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
+                        doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
+                        indexWriter.AddDocument(doc);
+                    }
+
+                    Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
+                }
+                for (var i = 0; i < 100; i++)
+                {
+                    using (var searcher = new IndexSearcher(azureReadWriteDirectory))
+                    {
+                        Assert.AreNotEqual(0, SearchForPhrase(searcher, "dog"));
+                        Assert.AreNotEqual(0, SearchForPhrase(searcher, "cat"));
+                        Assert.AreNotEqual(0, SearchForPhrase(searcher, "car"));
+                    }
+                }
+
+                readonlyDirectoryFolder.Create();
+                azureReadOnlyDirectory = new AzureReadOnlyLuceneDirectory(connectionString, containerName, readonlyDirectoryFolder.FullName, containerName);
+                azureReadOnlyDirectory.SetMergePolicyAction(e => new NoMergePolicy(e));
+                azureReadOnlyDirectory.SetMergeScheduler(new NoMergeSheduler());
+                azureReadOnlyDirectory.SetDeletion(new NoDeletionPolicy());
+
+                using (var indexWriter = new IndexWriter(azureReadOnlyDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),
+                   !IndexReader.IndexExists(azureReadOnlyDirectory), azureReadOnlyDirectory.GetDeletionPolicy(),
+                   new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
+                {
+
+                    indexWriter.SetRAMBufferSizeMB(10.0);
+                    indexWriter.SetMergePolicy(azureReadOnlyDirectory.GetMergePolicy(indexWriter));
+                    indexWriter.SetMergeScheduler(azureReadOnlyDirectory.GetMergeScheduler());
+                    //TODO: Use Test index. adding documents should be blocked
+                }
+                for (var i = 0; i < 100; i++)
+                {
+                    using (var searcher = new IndexSearcher(azureReadOnlyDirectory))
+                    {
+                        Assert.AreEqual(0, SearchForPhrase(searcher, "Sony"));
+                        Assert.AreEqual(0, SearchForPhrase(searcher, "Nintendo"));
+                    }
+                }
+                //Use 
+            }
+            finally
+            {
+                readWriteCacheDirectory?.Dispose();
+                azureReadOnlyDirectory?.Dispose();
+                azureReadWriteDirectory?.Dispose();
+                // check the container exists, and delete it
+                var containerClient = new BlobContainerClient(connectionString, containerName);
+                var exists = containerClient.Exists();
+                if (exists)
+                    containerClient.Delete();
+
+                if (readonlyDirectoryFolder.Exists)
+                {
+                    readonlyDirectoryFolder.Delete(true);
+                }
+            }
+
+        }
+
         static int SearchForPhrase(IndexSearcher searcher, string phrase)
         {
             var parser = new Lucene.Net.QueryParsers.QueryParser(Lucene.Net.Util.Version.LUCENE_CURRENT, "Body", new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT));