generated protobuf

Author: rekby

Makefile

@@ -0,0 +1,3 @@
+protobuf:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env python generate_protoc.py

generate-protobuf.Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.9.15
+RUN \
+    python -m pip install --upgrade pip && \
+    python -m pip install grpcio==1.39.0 && \
+    python -m pip install grpclib && \
+    python -m pip install protobuf==3.20.3 && \
+    python -m pip install grpcio-tools==1.38.0
+
+ENV PROTOC_VER=21.8
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip && \
+    unzip protoc-*.zip && \
+    rm -f protoc-*.zip && \
+    mv bin/protoc /usr/local/bin/protoc && \
+    mv include well-known-protos

generate_protoc.py

@@ -23,30 +23,43 @@ def files_filter(dir, items: List[str]) -> List[str]:
     return ignore
 
 
-def create_init_files(dir):
-    for root, _dirs, _files in os.walk(dir):
+def create_init_files(rootdirpath: str):
+    for root, _dirs, _files in os.walk(rootdirpath):
         init_path = pathlib.Path(os.path.join(root, '__init__.py'))
         if not init_path.exists():
             init_path.touch()
 
 
-def replace_protos_dir():
-    def replace_dir(src, dst):
-        shutil.rmtree(dst, ignore_errors=True)
-        shutil.copytree(src, dst, ignore=files_filter)
-        create_init_files(dst)
-
-    replace_dir("ydb-api-protos", "ydb/_grpc")
-
-
-def remove_protos():
-    for root, _dirs, files in os.walk("ydb/public/api"):
+def remove_protos(rootdirpath: str):
+    for root, _dirs, files in os.walk(rootdirpath):
         for file in files:
             if file.endswith(".proto"):
                 os.remove(os.path.join(root, file))
 
 
+def fiximports(rootdir: str):
+    for dirpath, _, fnames in os.walk(rootdir):
+        for fname in fnames:
+            if not fname.endswith(".py"):
+                continue
+
+            with open(os.path.join(dirpath, fname), 'r+t') as f:
+                content = f.read()
+                content = content.replace("from protos", "from ydb._grpc.protos")
+                f.seek(0)
+                f.write(content)
+
+
+def generate_protobuf(src_proto_dir: str, dst_dir: str):
+    shutil.rmtree(dst_dir, ignore_errors=True)
+
+    shutil.copytree(src_proto_dir, dst_dir, ignore=files_filter)
+    create_init_files(dst_dir)
+
+    command.build_package_protos(dst_dir)
+    remove_protos(dst_dir)
+    fiximports(dst_dir)
+
+
 if __name__ == '__main__':
-    replace_protos_dir()
-    command.build_package_protos("ydb/_grpc")
-    # remove_protos()
+    generate_protobuf("ydb-api-protos", "ydb/_grpc")

ydb/__init__.py

@@ -7,7 +7,6 @@
 from .settings import *  # noqa
 from .resolver import *  # noqa
 from .export import *  # noqa
-from .s3list import *  # noqa
 from .auth_helpers import *  # noqa
 from .operation import *  # noqa
 from .scripting import *  # noqa

ydb/public/api/protos/annotations/__init__.py renamed to ydb/_grpc/__init__.py

@@ -2,7 +2,7 @@
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
 
-from protos import ydb_auth_pb2 as protos_dot_ydb__auth__pb2
+from ydb._grpc.protos import ydb_auth_pb2 as protos_dot_ydb__auth__pb2
 
 
 class AuthServiceStub(object):

ydb/_grpc/protos/__init__.py

@@ -2,7 +2,7 @@
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
 
-from protos import ydb_cms_pb2 as protos_dot_ydb__cms__pb2
+from ydb._grpc.protos import ydb_cms_pb2 as protos_dot_ydb__cms__pb2
 
 
 class CmsServiceStub(object):