add test for azure

Author: vnghia

tests/test_fs_plugins.py

@@ -125,7 +125,52 @@ def exists(key_name):
     return get_uri, read_file, write_file, exists
 
 
-MAP_URI_HELPER = {S3_URI: s3_init, GCS_URI: gcs_init}
+def az_init(monkeypatch):
+    from azure.storage.blob import ContainerClient
+    from azure.core.exceptions import ResourceNotFoundError
+
+    container_name = os.environ.get("AZ_TEST_CONTAINER")
+    account = ""
+    prefix = f"tf-io-dir-{int(time.time())}/"
+    client = None
+    if container_name is None:
+        # This means we are running against emulator.
+        monkeypatch.setenv("TF_AZURE_USE_DEV_STORAGE", "1")
+        container_name = "tf-io-{}-{}".format(AZ_URI, int(time.time()))
+        account = "devstoreaccount1"
+        conn_str = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+        client = ContainerClient.from_connection_string(
+            conn_str=conn_str, container_name=container_name
+        )
+        client.create_container()
+    else:
+        # TODO(vnvo2409): Implement this for testing against production
+        pass
+
+    client.upload_blob(prefix, b"")
+
+    def get_uri(key_name):
+        return "{}://{}/{}/{}".format(
+            AZ_URI, account, container_name, prefix + key_name
+        )
+
+    def read_file(key_name):
+        return client.download_blob(prefix + key_name).content_as_bytes()
+
+    def write_file(key_name, body):
+        client.upload_blob(prefix + key_name, body)
+
+    def exists(key_name):
+        try:
+            client.download_blob(prefix + key_name).content_as_bytes()
+        except ResourceNotFoundError:
+            return False
+        return True
+
+    return get_uri, read_file, write_file, exists
+
+
+MAP_URI_HELPER = {S3_URI: s3_init, GCS_URI: gcs_init, AZ_URI: az_init}
 
 
 # ------------------------ URI CONDITION FOR EACH TEST ----------------------- #
@@ -188,7 +233,7 @@ def check_test_condition_and_setup_env(uri, envs, monkeypatch):
     autouse=True,
     params=[
         S3_URI,
-        pytest.param(AZ_URI, marks=pytest.mark.skip(reason="TODO")),
+        AZ_URI,
         pytest.param(HDFS_URI, marks=pytest.mark.skip(reason="TODO")),
         pytest.param(VIEWFS_URI, marks=pytest.mark.skip(reason="TODO")),
         pytest.param(HAR_URI, marks=pytest.mark.skip(reason="TODO")),
@@ -262,11 +307,12 @@ def test_gfile_GFile_readable(uri_init, envs, monkeypatch):
         assert file_read == body
 
     # Notfound
-    with pytest.raises(tf.errors.NotFoundError) as excinfo:
-        fname_not_found = fname + "_not_found"
-        with tf.io.gfile.GFile(fname_not_found, "rb") as f:
-            _ = f.read()
-        assert fname_not_found in str(excinfo.value)
+    # TODO(vnvo2409): `az` returns an empty string when file does not exist.
+    if uri != AZ_URI:
+        with pytest.raises(tf.errors.NotFoundError):
+            fname_not_found = fname + "_not_found"
+            with tf.io.gfile.GFile(fname_not_found, "rb") as f:
+                _ = f.read()
 
     # Read length
     with tf.io.gfile.GFile(fname, "rb") as f:
@@ -325,10 +371,12 @@ def test_gfile_GFile_writable(uri_init, envs, monkeypatch):
         assert read_file(base_file_name) == body
 
     # Append
-    with tf.io.gfile.GFile(fname, "ab") as f:
-        f.write(base_body)
-        f.flush()
-        assert read_file(base_file_name) == body + base_body
+    # `NewAppendableFile` in `az` is actually `NewWritableFile`
+    if uri != AZ_URI:
+        with tf.io.gfile.GFile(fname, "ab") as f:
+            f.write(base_body)
+            f.flush()
+            assert read_file(base_file_name) == body + base_body
 
 
 @pytest.mark.parametrize("envs", [("+all", None)])
@@ -341,7 +389,9 @@ def test_gfile_isdir(uri_init, envs, monkeypatch):
     base_file_name = base_dir_name + "fname"
     file_name = get_uri(base_file_name)
 
-    write_file(base_dir_name, "")
+    # `az` considers non-file path as a directory
+    if uri != AZ_URI:
+        write_file(base_dir_name, "")
     write_file(base_file_name, "")
 
     assert tf.io.gfile.isdir(dir_name) is True
@@ -355,7 +405,9 @@ def test_gfile_listdir(uri_init, envs, monkeypatch):
 
     base_dir_name = "test_gfile_listdir/"
     dir_name = get_uri(base_dir_name)
-    write_file(base_dir_name, "")
+    # `az` considers non-file path as a directory
+    if uri != AZ_URI:
+        write_file(base_dir_name, "")
 
     num_childs = 5
     childrens = [None] * num_childs
@@ -416,7 +468,8 @@ def test_gfile_rmtree(uri_init, envs, monkeypatch):
     assert [exists(get_uri(entry)) for entry in base_names] == [False] * num_entries
 
 
-@pytest.mark.parametrize("envs", [("+all", None)])
+# TODO(vnvo2409): `az` copy operations causes an infinite loop.
+@pytest.mark.parametrize("envs", [("+all:-az", None)])
 def test_gfile_copy(uri_init, envs, monkeypatch):
     uri, get_uri, read_file, write_file, _ = uri_init
     check_test_condition_and_setup_env(uri, envs, monkeypatch)