List invalid xml (Azure#25092)

* Added service version. Updated sas recordings

* Updated datalake version conversion

* Updated customizations

* Generated new code

* Added code for new feature

* Added tests

* Moved BlobName to models package

* Created BlobPrefixInternal

* Added last test recordings

* Fixed some ci errors

* Hopefully ci fixes

* Updated recordings

* reverted swagger to main branch

* Updated changelog

Author: rickle-msft

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 12.15.0-beta.2 (Unreleased)
 
 ### Features Added
+- Added support for the 2021-02-12 service version.
+- Added support for listing blobs which contain invalid xml characters.
 
 ### Breaking Changes
 

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java

@@ -1022,7 +1022,8 @@ PagedFlux<BlobItem> listBlobsHierarchyWithOptionalTimeout(String delimiter, List
                         : Stream.concat(
                         response.getValue().getSegment().getBlobItems().stream().map(ModelHelper::populateBlobItem),
                         response.getValue().getSegment().getBlobPrefixes().stream()
-                            .map(blobPrefix -> new BlobItem().setName(blobPrefix.getName()).setIsPrefix(true))
+                            .map(blobPrefix -> new BlobItem()
+                                .setName(ModelHelper.toBlobNameString(blobPrefix.getName())).setIsPrefix(true))
                     ).collect(Collectors.toList());
 
                     return new PagedResponseBase<>(

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/AzureBlobStorageImplBuilder.java

@@ -212,7 +212,7 @@ public AzureBlobStorageImplBuilder addPolicy(HttpPipelinePolicy customPolicy) {
      */
     public AzureBlobStorageImpl buildClient() {
         if (version == null) {
-            this.version = "2020-12-06";
+            this.version = "2021-02-12";
         }
         if (pipeline == null) {
             this.pipeline = createHttpPipeline();

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobHierarchyListSegment.java

@@ -5,7 +5,6 @@
 package com.azure.storage.blob.implementation.models;
 
 import com.azure.core.annotation.Fluent;
-import com.azure.storage.blob.models.BlobPrefix;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
@@ -21,7 +20,7 @@ public final class BlobHierarchyListSegment {
      * The BlobPrefixes property.
      */
     @JsonProperty("BlobPrefix")
-    private List<BlobPrefix> blobPrefixes = new ArrayList<>();
+    private List<BlobPrefixInternal> blobPrefixes = new ArrayList<>();
 
     /*
      * The BlobItems property.
@@ -34,7 +33,7 @@ public final class BlobHierarchyListSegment {
      *
      * @return the blobPrefixes value.
      */
-    public List<BlobPrefix> getBlobPrefixes() {
+    public List<BlobPrefixInternal> getBlobPrefixes() {
         return this.blobPrefixes;
     }
 
@@ -44,7 +43,7 @@ public List<BlobPrefix> getBlobPrefixes() {
      * @param blobPrefixes the blobPrefixes value to set.
      * @return the BlobHierarchyListSegment object itself.
      */
-    public BlobHierarchyListSegment setBlobPrefixes(List<BlobPrefix> blobPrefixes) {
+    public BlobHierarchyListSegment setBlobPrefixes(List<BlobPrefixInternal> blobPrefixes) {
         this.blobPrefixes = blobPrefixes;
         return this;
     }

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobItemInternal.java

@@ -17,7 +17,7 @@ public final class BlobItemInternal {
      * The Name property.
      */
     @JsonProperty(value = "Name", required = true)
-    private String name;
+    private BlobName name;
 
     /*
      * The Deleted property.
@@ -84,7 +84,7 @@ public final class BlobItemInternal {
      *
      * @return the name value.
      */
-    public String getName() {
+    public BlobName getName() {
         return this.name;
     }
 
@@ -94,7 +94,7 @@ public String getName() {
      * @param name the name value to set.
      * @return the BlobItemInternal object itself.
      */
-    public BlobItemInternal setName(String name) {
+    public BlobItemInternal setName(BlobName name) {
         this.name = name;
         return this;
     }

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobName.java

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+// Code generated by Microsoft (R) AutoRest Code Generator.
+
+package com.azure.storage.blob.implementation.models;
+
+import com.azure.core.annotation.Fluent;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlText;
+
+/** The BlobName model. */
+@JacksonXmlRootElement(localName = "BlobName")
+@Fluent
+public final class BlobName {
+    /*
+     * Indicates if the blob name is encoded.
+     */
+    @JacksonXmlProperty(localName = "Encoded", isAttribute = true)
+    private Boolean encoded;
+
+    /*
+     * The name of the blob.
+     */
+    @JacksonXmlText
+    private String content;
+
+    /**
+     * Get the encoded property: Indicates if the blob name is encoded.
+     *
+     * @return the encoded value.
+     */
+    public Boolean isEncoded() {
+        return this.encoded;
+    }
+
+    /**
+     * Set the encoded property: Indicates if the blob name is encoded.
+     *
+     * @param encoded the encoded value to set.
+     * @return the BlobName object itself.
+     */
+    public BlobName setEncoded(Boolean encoded) {
+        this.encoded = encoded;
+        return this;
+    }
+
+    /**
+     * Get the content property: The name of the blob.
+     *
+     * @return the content value.
+     */
+    public String getContent() {
+        return this.content;
+    }
+
+    /**
+     * Set the content property: The name of the blob.
+     *
+     * @param content the content value to set.
+     * @return the BlobName object itself.
+     */
+    public BlobName setContent(String content) {
+        this.content = content;
+        return this;
+    }
+}

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobPrefixInternal.java

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+// Code generated by Microsoft (R) AutoRest Code Generator.
+
+package com.azure.storage.blob.implementation.models;
+
+import com.azure.core.annotation.Fluent;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
+
+/** The BlobPrefix model. */
+@JacksonXmlRootElement(localName = "BlobPrefix")
+@Fluent
+public final class BlobPrefixInternal {
+    /*
+     * The Name property.
+     */
+    @JsonProperty(value = "Name", required = true)
+    private BlobName name;
+
+    /**
+     * Get the name property: The Name property.
+     *
+     * @return the name value.
+     */
+    public BlobName getName() {
+        return this.name;
+    }
+
+    /**
+     * Set the name property: The Name property.
+     *
+     * @param name the name value to set.
+     * @return the BlobPrefix object itself.
+     */
+    public BlobPrefixInternal setName(BlobName name) {
+        this.name = name;
+        return this;
+    }
+}

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/CustomHierarchicalListingDeserializer.java

@@ -5,7 +5,7 @@
 
 import com.azure.storage.blob.implementation.models.BlobHierarchyListSegment;
 import com.azure.storage.blob.implementation.models.BlobItemInternal;
-import com.azure.storage.blob.models.BlobPrefix;
+import com.azure.storage.blob.implementation.models.BlobPrefixInternal;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.DeserializationContext;
@@ -26,12 +26,12 @@ public BlobHierarchyListSegment getNullValue(DeserializationContext ctxt) {
     @Override
     public BlobHierarchyListSegment deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
         ArrayList<BlobItemInternal> blobItems = new ArrayList<>();
-        ArrayList<BlobPrefix> blobPrefixes = new ArrayList<>();
+        ArrayList<BlobPrefixInternal> blobPrefixes = new ArrayList<>();
 
         JsonDeserializer<Object> blobItemDeserializer =
             ctxt.findRootValueDeserializer(ctxt.constructType(BlobItemInternal.class));
         JsonDeserializer<Object> blobPrefixDeserializer =
-            ctxt.findRootValueDeserializer(ctxt.constructType(BlobPrefix.class));
+            ctxt.findRootValueDeserializer(ctxt.constructType(BlobPrefixInternal.class));
 
         for (JsonToken currentToken = p.nextToken(); !currentToken.name().equals("END_OBJECT");
              currentToken = p.nextToken()) {
@@ -41,7 +41,7 @@ public BlobHierarchyListSegment deserialize(JsonParser p, DeserializationContext
             if (p.getCurrentName().equals("Blob")) {
                 blobItems.add((BlobItemInternal) blobItemDeserializer.deserialize(p, ctxt));
             } else if (p.getCurrentName().equals("BlobPrefix")) {
-                blobPrefixes.add((BlobPrefix) blobPrefixDeserializer.deserialize(p, ctxt));
+                blobPrefixes.add((BlobPrefixInternal) blobPrefixDeserializer.deserialize(p, ctxt));
             }
         }
 

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java

@@ -12,6 +12,7 @@
 import com.azure.storage.blob.ProgressReceiver;
 import com.azure.storage.blob.implementation.models.BlobItemInternal;
 import com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal;
+import com.azure.storage.blob.implementation.models.BlobName;
 import com.azure.storage.blob.implementation.models.BlobTag;
 import com.azure.storage.blob.implementation.models.BlobTags;
 import com.azure.storage.blob.implementation.models.BlobsDownloadHeaders;
@@ -31,6 +32,7 @@
 import com.azure.storage.blob.models.PageBlobCopyIncrementalRequestConditions;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.models.TaggedBlobItem;
+import com.azure.storage.common.Utility;
 import com.azure.storage.common.implementation.Constants;
 
 import java.io.IOException;
@@ -262,7 +264,7 @@ public static BlobDownloadHeaders populateBlobDownloadHeaders(
      */
     public static BlobItem populateBlobItem(BlobItemInternal blobItemInternal) {
         BlobItem blobItem = new BlobItem();
-        blobItem.setName(blobItemInternal.getName());
+        blobItem.setName(toBlobNameString(blobItemInternal.getName()));
         blobItem.setDeleted(blobItemInternal.isDeleted());
         blobItem.setSnapshot(blobItemInternal.getSnapshot());
         blobItem.setProperties(populateBlobItemProperties(blobItemInternal.getProperties()));
@@ -281,6 +283,12 @@ public static BlobItem populateBlobItem(BlobItemInternal blobItemInternal) {
         return blobItem;
     }
 
+    public static String toBlobNameString(BlobName blobName) {
+        return blobName.isEncoded() != null && blobName.isEncoded()
+            ? Utility.urlDecode(blobName.getContent())
+            : blobName.getContent();
+    }
+
     public static TaggedBlobItem populateTaggedBlobItem(FilterBlobItem filterBlobItem) {
         return new TaggedBlobItem(filterBlobItem.getContainerName(), filterBlobItem.getName(),
             tagMapFromBlobTags(filterBlobItem.getTags()));

sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAPITest.groovy

@@ -1067,6 +1067,19 @@ class ContainerAPITest extends APISpec {
         RehydratePriority.HIGH     || _
     }
 
+    @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "V2021_02_12")
+    def "List blobs flat invalid xml"() {
+        setup:
+        def blobName = "dir1/dir2/file\uFFFE.blob";
+        cc.getBlobClient(blobName).getAppendBlobClient().create()
+
+        when:
+        def blobItem = cc.listBlobs().iterator().next()
+
+        then:
+        blobItem.getName() == blobName
+    }
+
     def "List blobs flat error"() {
         setup:
         cc = primaryBlobServiceClient.getBlobContainerClient(generateContainerName())
@@ -1120,6 +1133,7 @@ class ContainerAPITest extends APISpec {
     This test requires two accounts that are configured in a very specific way. It is not feasible to setup that
     relationship programmatically, so we have recorded a successful interaction and only test recordings.
     */
+
     @PlaybackOnly
     def "List blobs flat ORS"() {
         setup:
@@ -1436,6 +1450,7 @@ class ContainerAPITest extends APISpec {
     This test requires two accounts that are configured in a very specific way. It is not feasible to setup that
     relationship programmatically, so we have recorded a successful interaction and only test recordings.
     */
+
     @PlaybackOnly
     def "List blobs hier ORS"() {
         setup:
@@ -1546,6 +1561,30 @@ class ContainerAPITest extends APISpec {
         blob.getProperties().isSealed()
     }
 
+    @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "V2021_02_12")
+    def "List blobs hier invalid xml"() {
+        setup:
+        def blobName = 'dir1/dir2/file\uFFFE.blob';
+        cc.getBlobClient(blobName).getAppendBlobClient().create()
+
+        when:
+        def blobItem
+        if (!delimiter) {
+            blobItem = cc.listBlobsByHierarchy("", null, null).iterator().next()
+        } else {
+            blobItem = cc.listBlobsByHierarchy(".b", null, null).iterator().next()
+        }
+
+        then:
+        blobItem.getName() == (delimiter ? "dir1/dir2/file\uFFFE.b" : blobName)
+        blobItem.isPrefix() == (delimiter ? true : null)
+
+        where:
+        delimiter | _
+        false     | _
+        true      | _
+    }
+
     def "List blobs hier error"() {
         setup:
         cc = primaryBlobServiceClient.getBlobContainerClient(generateContainerName())
@@ -1583,7 +1622,7 @@ class ContainerAPITest extends APISpec {
 
         where:
         name                  | _
-        "中文"                | _
+        "中文"                  | _
         "az[]"                | _
         "hello world"         | _
         "hello/world"         | _