From 8e122b895fc03944a02fa73338c801117034bc18 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 11 Jan 2023 21:07:16 -0600 Subject: [PATCH 01/21] Implemented pack_bytes and pack_command. --- hiredis/__init__.py | 9 +++- hiredis/hiredis.pyi | 8 +++- src/hiredis.c | 38 ++++++++++++++- src/pack.c | 111 ++++++++++++++++++++++++++++++++++++++++++++ src/pack.h | 9 ++++ 5 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 src/pack.c create mode 100644 src/pack.h diff --git a/hiredis/__init__.py b/hiredis/__init__.py index f01b169e..c145ea3f 100644 --- a/hiredis/__init__.py +++ b/hiredis/__init__.py @@ -1,6 +1,11 @@ -from .hiredis import Reader, HiredisError, ProtocolError, ReplyError +from .hiredis import Reader, HiredisError, pack_bytes, pack_command, ProtocolError, ReplyError from .version import __version__ __all__ = [ - "Reader", "HiredisError", "ProtocolError", "ReplyError", + "Reader", + "HiredisError", + "pack_bytes", + "pack_command", + "ProtocolError", + "ReplyError", "__version__"] diff --git a/hiredis/hiredis.pyi b/hiredis/hiredis.pyi index 61c0346b..5d9d1e53 100644 --- a/hiredis/hiredis.pyi +++ b/hiredis/hiredis.pyi @@ -1,4 +1,4 @@ -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional, Union, Tuple class HiredisError(Exception): ... class ProtocolError(HiredisError): ... @@ -24,3 +24,9 @@ class Reader: def set_encoding( self, encoding: Optional[str] = ..., errors: Optional[str] = ... ) -> None: ... + + +def pack_bytes(bytes: bytes): ... + + +def pack_command(cmd: Tuple[str|int|float|bytes]): ... diff --git a/src/hiredis.c b/src/hiredis.c index 9a0c88b2..a1ca2b0e 100644 --- a/src/hiredis.c +++ b/src/hiredis.c @@ -1,5 +1,6 @@ #include "hiredis.h" #include "reader.h" +#include "pack.h" static int hiredis_ModuleTraverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GET_STATE(m)->HiErr_Base); @@ -15,12 +16,47 @@ static int hiredis_ModuleClear(PyObject *m) { return 0; } +static PyObject* +py_pack_command(PyObject* self, PyObject* cmd) +{ + return pack_command(cmd); +} + +static PyObject* +py_pack_bytes(PyObject* self, PyObject* cmd) +{ + return pack_bytes(cmd); +} + +PyDoc_STRVAR(pack_command_doc, "Pack ...... "); +PyDoc_STRVAR(pack_bytes_doc, "Pack ...... "); + +PyMethodDef pack_command_method = { + "pack_command", /* The name as a C string. */ + (PyCFunction) py_pack_command, /* The C function to invoke. */ + METH_O, /* Flags telling Python how to invoke */ + pack_command_doc, /* The docstring as a C string. */ +}; + +PyMethodDef pack_bytes_method = { + "pack_bytes", /* The name as a C string. */ + (PyCFunction) py_pack_bytes, /* The C function to invoke. */ + METH_O, /* Flags telling Python how to invoke */ + pack_bytes_doc, /* The docstring as a C string. */ +}; + +PyMethodDef methods[] = { + {"pack_command", (PyCFunction) py_pack_command, METH_O, pack_command_doc}, + {"pack_bytes", (PyCFunction) py_pack_bytes, METH_O, pack_bytes_doc}, + {NULL}, +}; + static struct PyModuleDef hiredis_ModuleDef = { PyModuleDef_HEAD_INIT, MOD_HIREDIS, NULL, sizeof(struct hiredis_ModuleState), /* m_size */ - NULL, /* m_methods */ + methods, /* m_methods */ NULL, /* m_reload */ hiredis_ModuleTraverse, /* m_traverse */ hiredis_ModuleClear, /* m_clear */ diff --git a/src/pack.c b/src/pack.c new file mode 100644 index 00000000..cf7ae206 --- /dev/null +++ b/src/pack.c @@ -0,0 +1,111 @@ +#include "pack.h" +#include + +static +sds add_to_buffer(sds curr_buff, char *cmd, uint len) +{ + assert(curr_buff); + assert(sds); + sds newbuf = sdscatlen(curr_buff, cmd, len); + if (newbuf == NULL) { + PyErr_NoMemory(); + return NULL; + } + + return newbuf; +} + +PyObject* +pack_command(PyObject* cmd) +{ + char *buf = sdsempty(); + assert(cmd); + char *obj_buf = NULL; + Py_ssize_t obj_len = 0; + + if (cmd == NULL || !PyTuple_Check(cmd)) { + PyErr_SetString(PyExc_TypeError, + "The argument must be a tuple of str, int, float or bytes."); + return NULL; + } + + if (buf == NULL) { + return PyErr_NoMemory(); + } + + for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) { + PyObject *item = PyTuple_GetItem(cmd, i); + sds temp_ptr = NULL; + if (PyBytes_Check(item)) { + // check result? + PyBytes_AsStringAndSize(item, &obj_buf, &obj_len); + temp_ptr = add_to_buffer(buf, obj_buf, obj_len); + if (temp_ptr == NULL) { + return NULL; + } + + buf = temp_ptr; + + } else if (PyUnicode_Check(item)) { + + obj_buf = PyUnicode_AsUTF8AndSize(item, &obj_len); + if (obj_buf == NULL) { + sds_free(buf); + return NULL; + } + + temp_ptr = add_to_buffer(buf, obj_buf, obj_len); + if (temp_ptr == NULL) { + return NULL; + } + + buf = temp_ptr; + } else if PyLong_Check(item) { + } else { + printf("not yet\n"); + } + + temp_ptr = add_to_buffer(buf, " ", 1); + if (temp_ptr == NULL) { + return NULL; + } + buf = temp_ptr; + } + + obj_len = redisFormatCommand(&obj_buf, buf); + + PyObject *result = PyBytes_FromStringAndSize(obj_buf, obj_len); + + hi_free(obj_buf); + //sds_free(buf); + + return result; +} + + +PyObject* +pack_bytes(PyObject* cmd) +{ + assert(cmd); + if (cmd == NULL || !PyBytes_Check(cmd)) { + PyErr_SetString(PyExc_TypeError, + "The argument must be a tuple of str, int, float or bytes."); + return NULL; + } + + char *obj_buf = NULL; + Py_ssize_t obj_len = 0; + + if(PyBytes_AsStringAndSize(cmd, &obj_buf, &obj_len) == -1) { + return NULL; + } + + char * str_result = NULL; + obj_len = redisFormatCommand(&str_result, obj_buf); + + PyObject *result = PyBytes_FromStringAndSize(str_result, obj_len); + + hi_free(str_result); + + return result; +} diff --git a/src/pack.h b/src/pack.h new file mode 100644 index 00000000..a6359864 --- /dev/null +++ b/src/pack.h @@ -0,0 +1,9 @@ +#ifndef __PACK_H +#define __PACK_H + +#include + +extern PyObject* pack_command(PyObject* cmd); +extern PyObject* pack_bytes(PyObject* bytes); + +#endif \ No newline at end of file From be557768b8db7e78243644f6b2b9e44c3c2af992 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Sat, 14 Jan 2023 04:52:54 -0600 Subject: [PATCH 02/21] Bump version + make it work in a clean environment --- hiredis/version.py | 2 +- setup.py | 5 +++-- src/pack.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) mode change 100755 => 100644 setup.py diff --git a/hiredis/version.py b/hiredis/version.py index 58039f50..4eabd0b3 100644 --- a/hiredis/version.py +++ b/hiredis/version.py @@ -1 +1 @@ -__version__ = "2.1.1" +__version__ = "2.1.2" diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 01677ce5..dbc0a93e --- a/setup.py +++ b/setup.py @@ -13,8 +13,9 @@ def version(): ext = Extension("hiredis.hiredis", sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "read", "sds")]), - extra_compile_args=["-std=c99"], + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + #glob.glob("vendor/hiredis/*.c")), + extra_compile_args=["-std=c99", "-O2"], include_dirs=["vendor"]) setup( diff --git a/src/pack.c b/src/pack.c index cf7ae206..ac6586ed 100644 --- a/src/pack.c +++ b/src/pack.c @@ -89,7 +89,7 @@ pack_bytes(PyObject* cmd) assert(cmd); if (cmd == NULL || !PyBytes_Check(cmd)) { PyErr_SetString(PyExc_TypeError, - "The argument must be a tuple of str, int, float or bytes."); + "The argument must be bytes."); return NULL; } From 2d45fb15829008445642c176b2c223aee782b2e0 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 16 Jan 2023 10:29:21 -0600 Subject: [PATCH 03/21] pack_command that passes redis-py test (*almost*) --- hiredis/hiredis.pyi | 2 +- src/pack.c | 160 ++++++++++++++++++++++++++------------------ 2 files changed, 96 insertions(+), 66 deletions(-) diff --git a/hiredis/hiredis.pyi b/hiredis/hiredis.pyi index 5d9d1e53..b318637b 100644 --- a/hiredis/hiredis.pyi +++ b/hiredis/hiredis.pyi @@ -29,4 +29,4 @@ class Reader: def pack_bytes(bytes: bytes): ... -def pack_command(cmd: Tuple[str|int|float|bytes]): ... +def pack_command(cmd: Tuple[str|int|float|bytes|memoryview]): ... diff --git a/src/pack.c b/src/pack.c index ac6586ed..6eb75d1a 100644 --- a/src/pack.c +++ b/src/pack.c @@ -1,106 +1,136 @@ #include "pack.h" #include +#include -static -sds add_to_buffer(sds curr_buff, char *cmd, uint len) +PyObject * +pack_command(PyObject *cmd) { - assert(curr_buff); - assert(sds); - sds newbuf = sdscatlen(curr_buff, cmd, len); - if (newbuf == NULL) { - PyErr_NoMemory(); - return NULL; - } - - return newbuf; -} - -PyObject* -pack_command(PyObject* cmd) -{ - char *buf = sdsempty(); assert(cmd); - char *obj_buf = NULL; - Py_ssize_t obj_len = 0; - - if (cmd == NULL || !PyTuple_Check(cmd)) { - PyErr_SetString(PyExc_TypeError, + PyObject *result = NULL; + + if (cmd == NULL || !PyTuple_Check(cmd)) + { + PyErr_SetString(PyExc_TypeError, "The argument must be a tuple of str, int, float or bytes."); return NULL; } - if (buf == NULL) { + int tokens_number = PyTuple_Size(cmd); + sds *tokens = s_malloc(sizeof(sds) * tokens_number); + if (tokens == NULL) + { + return PyErr_NoMemory(); + } + memset(tokens, 0, sizeof(sds) * tokens_number); + + size_t *lenghts = hi_malloc(sizeof(size_t) * tokens_number); + Py_ssize_t len = 0; + if (lenghts == NULL) + { + sds_free(tokens); return PyErr_NoMemory(); } - for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) { + for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) + { PyObject *item = PyTuple_GetItem(cmd, i); - sds temp_ptr = NULL; - if (PyBytes_Check(item)) { + + if (PyBytes_Check(item)) + { + // Does it need to PyObject_CheckBuffer + Py_buffer buffer; + PyObject_GetBuffer(item, &buffer, PyBUF_SIMPLE); + char *bytes = NULL; // check result? - PyBytes_AsStringAndSize(item, &obj_buf, &obj_len); - temp_ptr = add_to_buffer(buf, obj_buf, obj_len); - if (temp_ptr == NULL) { - return NULL; + PyBytes_AsStringAndSize(item, &bytes, &len); + tokens[i] = sdsempty(); + tokens[i] = sdscpylen(tokens[i], bytes, len); + lenghts[i] = buffer.len; + PyBuffer_Release(&buffer); + } + else if (PyUnicode_Check(item)) + { + const char *bytes = PyUnicode_AsUTF8AndSize(item, &len); + if (bytes == NULL) + { + // PyUnicode_AsUTF8AndSize sets an exception. + goto cleanup; } - buf = temp_ptr; - - } else if (PyUnicode_Check(item)) { - - obj_buf = PyUnicode_AsUTF8AndSize(item, &obj_len); - if (obj_buf == NULL) { - sds_free(buf); - return NULL; + tokens[i] = sdsnewlen(bytes, len); + lenghts[i] = len; + } + // else if (PyByteArray_Check(item)) + // { + // //not tested. Is it supported + // tokens[i] = sdsnewlen(PyByteArray_AS_STRING(item), PyByteArray_GET_SIZE(item)); + // lenghts[i] = PyByteArray_GET_SIZE(item); + // } + else if (PyMemoryView_Check(item)) + { + Py_buffer *p_buf = PyMemoryView_GET_BUFFER(item); + tokens[i] = sdsnewlen(p_buf->buf, p_buf->len); + lenghts[i] = p_buf->len; + } + else + { + if (PyLong_CheckExact(item) || PyFloat_Check(item)) + { + PyObject *repr = PyObject_Repr(item); + const char *bytes = PyUnicode_AsUTF8AndSize(repr, &len); + + tokens[i] = sdsnewlen(bytes, len); + lenghts[i] = len; + Py_DECREF(repr); } - - temp_ptr = add_to_buffer(buf, obj_buf, obj_len); - if (temp_ptr == NULL) { - return NULL; + else + { + PyErr_SetString(PyExc_TypeError, + "A tuple item must be str, int, float or bytes."); + goto cleanup; } - - buf = temp_ptr; - } else if PyLong_Check(item) { - } else { - printf("not yet\n"); - } - - temp_ptr = add_to_buffer(buf, " ", 1); - if (temp_ptr == NULL) { - return NULL; } - buf = temp_ptr; } - obj_len = redisFormatCommand(&obj_buf, buf); + char *resp_bytes = NULL; - PyObject *result = PyBytes_FromStringAndSize(obj_buf, obj_len); + len = redisFormatCommandArgv(&resp_bytes, tokens_number, tokens, lenghts); - hi_free(obj_buf); - //sds_free(buf); + if (len == -1) + { + PyErr_SetString(PyExc_RuntimeError, + "Failed to serialize the command."); + goto cleanup; + } + result = PyBytes_FromStringAndSize(resp_bytes, len); + hi_free(resp_bytes); +cleanup: + sdsfreesplitres(tokens, tokens_number); + hi_free(lenghts); return result; } - -PyObject* -pack_bytes(PyObject* cmd) +PyObject * +pack_bytes(PyObject *cmd) { assert(cmd); - if (cmd == NULL || !PyBytes_Check(cmd)) { - PyErr_SetString(PyExc_TypeError, + if (cmd == NULL || !PyBytes_Check(cmd)) + { + PyErr_SetString(PyExc_TypeError, "The argument must be bytes."); return NULL; } - char *obj_buf = NULL; + char *obj_buf = NULL; Py_ssize_t obj_len = 0; - if(PyBytes_AsStringAndSize(cmd, &obj_buf, &obj_len) == -1) { + if (PyBytes_AsStringAndSize(cmd, &obj_buf, &obj_len) == -1) + { return NULL; } - char * str_result = NULL; + char *str_result = NULL; obj_len = redisFormatCommand(&str_result, obj_buf); PyObject *result = PyBytes_FromStringAndSize(str_result, obj_len); From fdb131466833fb787938bfebc5dacc3a23d20020 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 18 Jan 2023 06:54:30 -0600 Subject: [PATCH 04/21] Removed pack-bytes and added some tests. --- hiredis/__init__.py | 3 +-- hiredis/hiredis.pyi | 22 +++++++++++++++------- src/hiredis.c | 16 +--------------- src/pack.c | 45 +++++---------------------------------------- src/pack.h | 1 - tests/test_pack.py | 24 ++++++++++++++++++++++++ 6 files changed, 46 insertions(+), 65 deletions(-) create mode 100644 tests/test_pack.py diff --git a/hiredis/__init__.py b/hiredis/__init__.py index c145ea3f..10afc025 100644 --- a/hiredis/__init__.py +++ b/hiredis/__init__.py @@ -1,10 +1,9 @@ -from .hiredis import Reader, HiredisError, pack_bytes, pack_command, ProtocolError, ReplyError +from .hiredis import Reader, HiredisError, pack_command, ProtocolError, ReplyError from .version import __version__ __all__ = [ "Reader", "HiredisError", - "pack_bytes", "pack_command", "ProtocolError", "ReplyError", diff --git a/hiredis/hiredis.pyi b/hiredis/hiredis.pyi index b318637b..d75e88d9 100644 --- a/hiredis/hiredis.pyi +++ b/hiredis/hiredis.pyi @@ -1,8 +1,17 @@ from typing import Any, Callable, Optional, Union, Tuple -class HiredisError(Exception): ... -class ProtocolError(HiredisError): ... -class ReplyError(HiredisError): ... + +class HiredisError(Exception): + ... + + +class ProtocolError(HiredisError): + ... + + +class ReplyError(HiredisError): + ... + class Reader: def __init__( @@ -13,6 +22,7 @@ class Reader: errors: Optional[str] = ..., notEnoughData: Any = ..., ) -> None: ... + def feed( self, __buf: Union[str, bytes], __off: int = ..., __len: int = ... ) -> None: ... @@ -21,12 +31,10 @@ class Reader: def getmaxbuf(self) -> int: ... def len(self) -> int: ... def has_data(self) -> bool: ... + def set_encoding( self, encoding: Optional[str] = ..., errors: Optional[str] = ... ) -> None: ... -def pack_bytes(bytes: bytes): ... - - -def pack_command(cmd: Tuple[str|int|float|bytes|memoryview]): ... +def pack_command(cmd: Tuple[str | int | float | bytes | memoryview]): ... diff --git a/src/hiredis.c b/src/hiredis.c index a1ca2b0e..c96097dc 100644 --- a/src/hiredis.c +++ b/src/hiredis.c @@ -22,14 +22,7 @@ py_pack_command(PyObject* self, PyObject* cmd) return pack_command(cmd); } -static PyObject* -py_pack_bytes(PyObject* self, PyObject* cmd) -{ - return pack_bytes(cmd); -} - -PyDoc_STRVAR(pack_command_doc, "Pack ...... "); -PyDoc_STRVAR(pack_bytes_doc, "Pack ...... "); +PyDoc_STRVAR(pack_command_doc, "Pack a series of arguments into the Redis protocol"); PyMethodDef pack_command_method = { "pack_command", /* The name as a C string. */ @@ -38,16 +31,9 @@ PyMethodDef pack_command_method = { pack_command_doc, /* The docstring as a C string. */ }; -PyMethodDef pack_bytes_method = { - "pack_bytes", /* The name as a C string. */ - (PyCFunction) py_pack_bytes, /* The C function to invoke. */ - METH_O, /* Flags telling Python how to invoke */ - pack_bytes_doc, /* The docstring as a C string. */ -}; PyMethodDef methods[] = { {"pack_command", (PyCFunction) py_pack_command, METH_O, pack_command_doc}, - {"pack_bytes", (PyCFunction) py_pack_bytes, METH_O, pack_bytes_doc}, {NULL}, }; diff --git a/src/pack.c b/src/pack.c index 6eb75d1a..eebcb8c5 100644 --- a/src/pack.c +++ b/src/pack.c @@ -21,27 +21,27 @@ pack_command(PyObject *cmd) { return PyErr_NoMemory(); } + memset(tokens, 0, sizeof(sds) * tokens_number); size_t *lenghts = hi_malloc(sizeof(size_t) * tokens_number); - Py_ssize_t len = 0; if (lenghts == NULL) { sds_free(tokens); return PyErr_NoMemory(); } + Py_ssize_t len = 0; + for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) { PyObject *item = PyTuple_GetItem(cmd, i); if (PyBytes_Check(item)) { - // Does it need to PyObject_CheckBuffer + char *bytes = NULL; Py_buffer buffer; PyObject_GetBuffer(item, &buffer, PyBUF_SIMPLE); - char *bytes = NULL; - // check result? PyBytes_AsStringAndSize(item, &bytes, &len); tokens[i] = sdsempty(); tokens[i] = sdscpylen(tokens[i], bytes, len); @@ -60,12 +60,6 @@ pack_command(PyObject *cmd) tokens[i] = sdsnewlen(bytes, len); lenghts[i] = len; } - // else if (PyByteArray_Check(item)) - // { - // //not tested. Is it supported - // tokens[i] = sdsnewlen(PyByteArray_AS_STRING(item), PyByteArray_GET_SIZE(item)); - // lenghts[i] = PyByteArray_GET_SIZE(item); - // } else if (PyMemoryView_Check(item)) { Py_buffer *p_buf = PyMemoryView_GET_BUFFER(item); @@ -109,33 +103,4 @@ pack_command(PyObject *cmd) sdsfreesplitres(tokens, tokens_number); hi_free(lenghts); return result; -} - -PyObject * -pack_bytes(PyObject *cmd) -{ - assert(cmd); - if (cmd == NULL || !PyBytes_Check(cmd)) - { - PyErr_SetString(PyExc_TypeError, - "The argument must be bytes."); - return NULL; - } - - char *obj_buf = NULL; - Py_ssize_t obj_len = 0; - - if (PyBytes_AsStringAndSize(cmd, &obj_buf, &obj_len) == -1) - { - return NULL; - } - - char *str_result = NULL; - obj_len = redisFormatCommand(&str_result, obj_buf); - - PyObject *result = PyBytes_FromStringAndSize(str_result, obj_len); - - hi_free(str_result); - - return result; -} +} \ No newline at end of file diff --git a/src/pack.h b/src/pack.h index a6359864..950193be 100644 --- a/src/pack.h +++ b/src/pack.h @@ -4,6 +4,5 @@ #include extern PyObject* pack_command(PyObject* cmd); -extern PyObject* pack_bytes(PyObject* bytes); #endif \ No newline at end of file diff --git a/tests/test_pack.py b/tests/test_pack.py new file mode 100644 index 00000000..91da540f --- /dev/null +++ b/tests/test_pack.py @@ -0,0 +1,24 @@ +import hiredis + + +def test_basic(): + cmd = ('HSET', 'foo', 'key', 'value1', b'key_b', b'bytes str', 'key_mv', + memoryview(b'bytes str'), b'key_i', 67, 'key_f', 3.14159265359) + expected_packed = b'*12\r\n$4\r\nHSET\r\n$3\r\nfoo\r\n$3\r\nkey\r\n$6\r\nvalue1\r\n$5\r\nkey_b\r\n$9\r\nbytes str\r\n$6\r\nkey_mv\r\n$9\r\nbytes str\r\n$5\r\nkey_i\r\n$2\r\n67\r\n$5\r\nkey_f\r\n$13\r\n3.14159265359\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed + + +def test_spaces_in_cmd(): + cmd = ('COMMAND', 'GETKEYS', 'EVAL', + 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}', 2, 'key1', 'key2', 'first', 'second') + expected_packed = b'*9\r\n$7\r\nCOMMAND\r\n$7\r\nGETKEYS\r\n$4\r\nEVAL\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$4\r\nkey1\r\n$4\r\nkey2\r\n$5\r\nfirst\r\n$6\r\nsecond\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed + + +def test_null_chars(): + cmd = ('SET', 'a', bytes(b'\xaa\x00\xffU')) + expected_packed = b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$4\r\n\xaa\x00\xffU\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed From 96e5d8e3e6711217f4bbe69890eefa29d99c06d4 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 18 Jan 2023 06:54:30 -0600 Subject: [PATCH 05/21] Removed pack-bytes and added some tests. --- hiredis/__init__.py | 3 +-- hiredis/hiredis.pyi | 22 +++++++++++++++------- setup.py | 3 +-- src/hiredis.c | 16 +--------------- src/pack.c | 45 +++++---------------------------------------- src/pack.h | 1 - tests/test_pack.py | 24 ++++++++++++++++++++++++ 7 files changed, 47 insertions(+), 67 deletions(-) create mode 100644 tests/test_pack.py diff --git a/hiredis/__init__.py b/hiredis/__init__.py index c145ea3f..10afc025 100644 --- a/hiredis/__init__.py +++ b/hiredis/__init__.py @@ -1,10 +1,9 @@ -from .hiredis import Reader, HiredisError, pack_bytes, pack_command, ProtocolError, ReplyError +from .hiredis import Reader, HiredisError, pack_command, ProtocolError, ReplyError from .version import __version__ __all__ = [ "Reader", "HiredisError", - "pack_bytes", "pack_command", "ProtocolError", "ReplyError", diff --git a/hiredis/hiredis.pyi b/hiredis/hiredis.pyi index b318637b..d75e88d9 100644 --- a/hiredis/hiredis.pyi +++ b/hiredis/hiredis.pyi @@ -1,8 +1,17 @@ from typing import Any, Callable, Optional, Union, Tuple -class HiredisError(Exception): ... -class ProtocolError(HiredisError): ... -class ReplyError(HiredisError): ... + +class HiredisError(Exception): + ... + + +class ProtocolError(HiredisError): + ... + + +class ReplyError(HiredisError): + ... + class Reader: def __init__( @@ -13,6 +22,7 @@ class Reader: errors: Optional[str] = ..., notEnoughData: Any = ..., ) -> None: ... + def feed( self, __buf: Union[str, bytes], __off: int = ..., __len: int = ... ) -> None: ... @@ -21,12 +31,10 @@ class Reader: def getmaxbuf(self) -> int: ... def len(self) -> int: ... def has_data(self) -> bool: ... + def set_encoding( self, encoding: Optional[str] = ..., errors: Optional[str] = ... ) -> None: ... -def pack_bytes(bytes: bytes): ... - - -def pack_command(cmd: Tuple[str|int|float|bytes|memoryview]): ... +def pack_command(cmd: Tuple[str | int | float | bytes | memoryview]): ... diff --git a/setup.py b/setup.py index dbc0a93e..b5e814cf 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension -import sys, importlib, os, glob, io +import importlib, glob, io def version(): loader = importlib.machinery.SourceFileLoader("hiredis.version", "hiredis/version.py") @@ -14,7 +14,6 @@ def version(): ext = Extension("hiredis.hiredis", sources=sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), - #glob.glob("vendor/hiredis/*.c")), extra_compile_args=["-std=c99", "-O2"], include_dirs=["vendor"]) diff --git a/src/hiredis.c b/src/hiredis.c index a1ca2b0e..c96097dc 100644 --- a/src/hiredis.c +++ b/src/hiredis.c @@ -22,14 +22,7 @@ py_pack_command(PyObject* self, PyObject* cmd) return pack_command(cmd); } -static PyObject* -py_pack_bytes(PyObject* self, PyObject* cmd) -{ - return pack_bytes(cmd); -} - -PyDoc_STRVAR(pack_command_doc, "Pack ...... "); -PyDoc_STRVAR(pack_bytes_doc, "Pack ...... "); +PyDoc_STRVAR(pack_command_doc, "Pack a series of arguments into the Redis protocol"); PyMethodDef pack_command_method = { "pack_command", /* The name as a C string. */ @@ -38,16 +31,9 @@ PyMethodDef pack_command_method = { pack_command_doc, /* The docstring as a C string. */ }; -PyMethodDef pack_bytes_method = { - "pack_bytes", /* The name as a C string. */ - (PyCFunction) py_pack_bytes, /* The C function to invoke. */ - METH_O, /* Flags telling Python how to invoke */ - pack_bytes_doc, /* The docstring as a C string. */ -}; PyMethodDef methods[] = { {"pack_command", (PyCFunction) py_pack_command, METH_O, pack_command_doc}, - {"pack_bytes", (PyCFunction) py_pack_bytes, METH_O, pack_bytes_doc}, {NULL}, }; diff --git a/src/pack.c b/src/pack.c index 6eb75d1a..eebcb8c5 100644 --- a/src/pack.c +++ b/src/pack.c @@ -21,27 +21,27 @@ pack_command(PyObject *cmd) { return PyErr_NoMemory(); } + memset(tokens, 0, sizeof(sds) * tokens_number); size_t *lenghts = hi_malloc(sizeof(size_t) * tokens_number); - Py_ssize_t len = 0; if (lenghts == NULL) { sds_free(tokens); return PyErr_NoMemory(); } + Py_ssize_t len = 0; + for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) { PyObject *item = PyTuple_GetItem(cmd, i); if (PyBytes_Check(item)) { - // Does it need to PyObject_CheckBuffer + char *bytes = NULL; Py_buffer buffer; PyObject_GetBuffer(item, &buffer, PyBUF_SIMPLE); - char *bytes = NULL; - // check result? PyBytes_AsStringAndSize(item, &bytes, &len); tokens[i] = sdsempty(); tokens[i] = sdscpylen(tokens[i], bytes, len); @@ -60,12 +60,6 @@ pack_command(PyObject *cmd) tokens[i] = sdsnewlen(bytes, len); lenghts[i] = len; } - // else if (PyByteArray_Check(item)) - // { - // //not tested. Is it supported - // tokens[i] = sdsnewlen(PyByteArray_AS_STRING(item), PyByteArray_GET_SIZE(item)); - // lenghts[i] = PyByteArray_GET_SIZE(item); - // } else if (PyMemoryView_Check(item)) { Py_buffer *p_buf = PyMemoryView_GET_BUFFER(item); @@ -109,33 +103,4 @@ pack_command(PyObject *cmd) sdsfreesplitres(tokens, tokens_number); hi_free(lenghts); return result; -} - -PyObject * -pack_bytes(PyObject *cmd) -{ - assert(cmd); - if (cmd == NULL || !PyBytes_Check(cmd)) - { - PyErr_SetString(PyExc_TypeError, - "The argument must be bytes."); - return NULL; - } - - char *obj_buf = NULL; - Py_ssize_t obj_len = 0; - - if (PyBytes_AsStringAndSize(cmd, &obj_buf, &obj_len) == -1) - { - return NULL; - } - - char *str_result = NULL; - obj_len = redisFormatCommand(&str_result, obj_buf); - - PyObject *result = PyBytes_FromStringAndSize(str_result, obj_len); - - hi_free(str_result); - - return result; -} +} \ No newline at end of file diff --git a/src/pack.h b/src/pack.h index a6359864..950193be 100644 --- a/src/pack.h +++ b/src/pack.h @@ -4,6 +4,5 @@ #include extern PyObject* pack_command(PyObject* cmd); -extern PyObject* pack_bytes(PyObject* bytes); #endif \ No newline at end of file diff --git a/tests/test_pack.py b/tests/test_pack.py new file mode 100644 index 00000000..91da540f --- /dev/null +++ b/tests/test_pack.py @@ -0,0 +1,24 @@ +import hiredis + + +def test_basic(): + cmd = ('HSET', 'foo', 'key', 'value1', b'key_b', b'bytes str', 'key_mv', + memoryview(b'bytes str'), b'key_i', 67, 'key_f', 3.14159265359) + expected_packed = b'*12\r\n$4\r\nHSET\r\n$3\r\nfoo\r\n$3\r\nkey\r\n$6\r\nvalue1\r\n$5\r\nkey_b\r\n$9\r\nbytes str\r\n$6\r\nkey_mv\r\n$9\r\nbytes str\r\n$5\r\nkey_i\r\n$2\r\n67\r\n$5\r\nkey_f\r\n$13\r\n3.14159265359\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed + + +def test_spaces_in_cmd(): + cmd = ('COMMAND', 'GETKEYS', 'EVAL', + 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}', 2, 'key1', 'key2', 'first', 'second') + expected_packed = b'*9\r\n$7\r\nCOMMAND\r\n$7\r\nGETKEYS\r\n$4\r\nEVAL\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$4\r\nkey1\r\n$4\r\nkey2\r\n$5\r\nfirst\r\n$6\r\nsecond\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed + + +def test_null_chars(): + cmd = ('SET', 'a', bytes(b'\xaa\x00\xffU')) + expected_packed = b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$4\r\n\xaa\x00\xffU\r\n' + packed_cmd = hiredis.pack_command(cmd) + assert packed_cmd == expected_packed From ff5ccaa0a1d110902184100682793b0da0d4efad Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 18 Jan 2023 07:59:21 -0600 Subject: [PATCH 06/21] even more clean up. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5e814cf..801eb509 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def version(): ext = Extension("hiredis.hiredis", sources=sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), - extra_compile_args=["-std=c99", "-O2"], + extra_compile_args=["-std=c99"], include_dirs=["vendor"]) setup( From 83edbe88ff7d979df9473bccf9c502c5f75cb149 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 19 Jan 2023 02:36:32 -0600 Subject: [PATCH 07/21] Re-factored tests and added a couple more ones. --- tests/test_pack.py | 55 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/test_pack.py b/tests/test_pack.py index 91da540f..2c3dbfd0 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -1,24 +1,47 @@ import hiredis +import pytest +testdata = [ + # all_types + (('HSET', 'foo', 'key', 'value1', b'key_b', b'bytes str', 'key_mv', memoryview(b'bytes str'), b'key_i', 67, 'key_f', 3.14159265359), + b'*12\r\n$4\r\nHSET\r\n$3\r\nfoo\r\n$3\r\nkey\r\n$6\r\nvalue1\r\n$5\r\nkey_b\r\n$9\r\nbytes str\r\n$6\r\nkey_mv\r\n$9\r\nbytes str\r\n$5\r\nkey_i\r\n$2\r\n67\r\n$5\r\nkey_f\r\n$13\r\n3.14159265359\r\n'), + # spaces_in_cmd + (('COMMAND', 'GETKEYS', 'EVAL', 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}', 2, 'key1', 'key2', 'first', 'second'), + b'*9\r\n$7\r\nCOMMAND\r\n$7\r\nGETKEYS\r\n$4\r\nEVAL\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$4\r\nkey1\r\n$4\r\nkey2\r\n$5\r\nfirst\r\n$6\r\nsecond\r\n'), + # null_chars + (('SET', 'a', bytes(b'\xaa\x00\xffU')), + b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$4\r\n\xaa\x00\xffU\r\n'), + # encoding + (('SET', 'a', "йדב문諺"), + b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$12\r\n\xD0\xB9\xD7\x93\xD7\x91\xEB\xAC\xB8\xE8\xAB\xBA\r\n'), + # big_int + (('SET', 'a', 2**128), + b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$39\r\n340282366920938463463374607431768211456\r\n'), +] -def test_basic(): - cmd = ('HSET', 'foo', 'key', 'value1', b'key_b', b'bytes str', 'key_mv', - memoryview(b'bytes str'), b'key_i', 67, 'key_f', 3.14159265359) - expected_packed = b'*12\r\n$4\r\nHSET\r\n$3\r\nfoo\r\n$3\r\nkey\r\n$6\r\nvalue1\r\n$5\r\nkey_b\r\n$9\r\nbytes str\r\n$6\r\nkey_mv\r\n$9\r\nbytes str\r\n$5\r\nkey_i\r\n$2\r\n67\r\n$5\r\nkey_f\r\n$13\r\n3.14159265359\r\n' - packed_cmd = hiredis.pack_command(cmd) - assert packed_cmd == expected_packed +testdata_ids = [ + "all_types", + "spaces_in_cmd", + "null_chars", + "encoding", + "big_int", +] -def test_spaces_in_cmd(): - cmd = ('COMMAND', 'GETKEYS', 'EVAL', - 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}', 2, 'key1', 'key2', 'first', 'second') - expected_packed = b'*9\r\n$7\r\nCOMMAND\r\n$7\r\nGETKEYS\r\n$4\r\nEVAL\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$4\r\nkey1\r\n$4\r\nkey2\r\n$5\r\nfirst\r\n$6\r\nsecond\r\n' +@pytest.mark.parametrize("cmd,expected_packed_cmd", testdata, ids=testdata_ids) +def test_basic(cmd, expected_packed_cmd): packed_cmd = hiredis.pack_command(cmd) - assert packed_cmd == expected_packed + assert packed_cmd == expected_packed_cmd -def test_null_chars(): - cmd = ('SET', 'a', bytes(b'\xaa\x00\xffU')) - expected_packed = b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$4\r\n\xaa\x00\xffU\r\n' - packed_cmd = hiredis.pack_command(cmd) - assert packed_cmd == expected_packed +def test_wrong_type(): + with pytest.raises(TypeError): + hiredis.pack_command(("HSET", "foo", True)) + + class Bar: + def __init__(self) -> None: + self.key = "key" + self.value = 36 + + with pytest.raises(TypeError): + hiredis.pack_command(("HSET", "foo", Bar())) From b54372da1af8790ba24f0b9edfdbb23b71152c97 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 19 Jan 2023 06:13:05 -0600 Subject: [PATCH 08/21] 1) Fix the compiler warning in pack.c 2) Fix a possible crash when running in Redis Gears. --- setup.py | 1 + src/pack.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 801eb509..d912699e 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ def version(): sources=sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), extra_compile_args=["-std=c99"], + extra_link_args=["-Wl,-Bsymbolic"], include_dirs=["vendor"]) setup( diff --git a/src/pack.c b/src/pack.c index eebcb8c5..42b2896b 100644 --- a/src/pack.c +++ b/src/pack.c @@ -88,7 +88,7 @@ pack_command(PyObject *cmd) char *resp_bytes = NULL; - len = redisFormatCommandArgv(&resp_bytes, tokens_number, tokens, lenghts); + len = redisFormatCommandArgv(&resp_bytes, tokens_number, (const char **)tokens, lenghts); if (len == -1) { From a5f8a3b74c6392c3073259b84050bbd3eba62bf2 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 10:44:31 -0600 Subject: [PATCH 09/21] 1) Fixed typo. 2) Reverted version bump. 3) Added an entry to the changelog. --- CHANGELOG.md | 2 ++ hiredis/version.py | 2 +- src/pack.c | 16 ++++++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ed1e79..2e577c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Implement pack_command that serializes redis-py command to the RESP bytes object. + ### 2.1.1 (2023-10-01) * Restores publishing of source distribution (#139) diff --git a/hiredis/version.py b/hiredis/version.py index 4eabd0b3..58039f50 100644 --- a/hiredis/version.py +++ b/hiredis/version.py @@ -1 +1 @@ -__version__ = "2.1.2" +__version__ = "2.1.1" diff --git a/src/pack.c b/src/pack.c index 42b2896b..0272a26f 100644 --- a/src/pack.c +++ b/src/pack.c @@ -24,8 +24,8 @@ pack_command(PyObject *cmd) memset(tokens, 0, sizeof(sds) * tokens_number); - size_t *lenghts = hi_malloc(sizeof(size_t) * tokens_number); - if (lenghts == NULL) + size_t *lengths = hi_malloc(sizeof(size_t) * tokens_number); + if (lengths == NULL) { sds_free(tokens); return PyErr_NoMemory(); @@ -45,7 +45,7 @@ pack_command(PyObject *cmd) PyBytes_AsStringAndSize(item, &bytes, &len); tokens[i] = sdsempty(); tokens[i] = sdscpylen(tokens[i], bytes, len); - lenghts[i] = buffer.len; + lengths[i] = buffer.len; PyBuffer_Release(&buffer); } else if (PyUnicode_Check(item)) @@ -58,13 +58,13 @@ pack_command(PyObject *cmd) } tokens[i] = sdsnewlen(bytes, len); - lenghts[i] = len; + lengths[i] = len; } else if (PyMemoryView_Check(item)) { Py_buffer *p_buf = PyMemoryView_GET_BUFFER(item); tokens[i] = sdsnewlen(p_buf->buf, p_buf->len); - lenghts[i] = p_buf->len; + lengths[i] = p_buf->len; } else { @@ -74,7 +74,7 @@ pack_command(PyObject *cmd) const char *bytes = PyUnicode_AsUTF8AndSize(repr, &len); tokens[i] = sdsnewlen(bytes, len); - lenghts[i] = len; + lengths[i] = len; Py_DECREF(repr); } else @@ -88,7 +88,7 @@ pack_command(PyObject *cmd) char *resp_bytes = NULL; - len = redisFormatCommandArgv(&resp_bytes, tokens_number, (const char **)tokens, lenghts); + len = redisFormatCommandArgv(&resp_bytes, tokens_number, (const char **)tokens, lengths); if (len == -1) { @@ -101,6 +101,6 @@ pack_command(PyObject *cmd) hi_free(resp_bytes); cleanup: sdsfreesplitres(tokens, tokens_number); - hi_free(lenghts); + hi_free(lengths); return result; } \ No newline at end of file From 241000c79d3f63b5b58e9fab1e11bc0655f9f2f7 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 10:44:31 -0600 Subject: [PATCH 10/21] 1) Fixed typo. 2) Reverted version bump. 3) Added an entry to the changelog. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d912699e..bc349b27 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def version(): ext = Extension("hiredis.hiredis", sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), extra_compile_args=["-std=c99"], extra_link_args=["-Wl,-Bsymbolic"], include_dirs=["vendor"]) From 4a416d7e56fc61f8a71baacbc84824d2d4ff0c5e Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 10:56:34 -0600 Subject: [PATCH 11/21] Autopep-8 on setup.py --- setup.py | 82 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/setup.py b/setup.py index bc349b27..27319752 100644 --- a/setup.py +++ b/setup.py @@ -4,54 +4,58 @@ from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension -import importlib, glob, io +import importlib +import glob +import io + def version(): loader = importlib.machinery.SourceFileLoader("hiredis.version", "hiredis/version.py") module = loader.load_module() return module.__version__ + ext = Extension("hiredis.hiredis", - sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), - extra_compile_args=["-std=c99"], - extra_link_args=["-Wl,-Bsymbolic"], - include_dirs=["vendor"]) + sources=sorted(glob.glob("src/*.c") + + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + extra_compile_args=["-std=c99"], + extra_link_args=["-Wl,-Bsymbolic"], + include_dirs=["vendor"]) setup( - name="hiredis", - version=version(), - description="Python wrapper for hiredis", - long_description=io.open('README.md', 'rt', encoding='utf-8').read(), - long_description_content_type='text/markdown', - url="https://github.com/redis/hiredis-py", - author="Jan-Erik Rediger, Pieter Noordhuis", - author_email="janerik@fnordig.de, pcnoordhuis@gmail.com", - keywords=["Redis"], - license="BSD", - packages=["hiredis"], - package_data={"hiredis": ["hiredis.pyi", "py.typed"]}, - ext_modules=[ext], - python_requires=">=3.7", - project_urls={ + name="hiredis", + version=version(), + description="Python wrapper for hiredis", + long_description=io.open('README.md', 'rt', encoding='utf-8').read(), + long_description_content_type='text/markdown', + url="https://github.com/redis/hiredis-py", + author="Jan-Erik Rediger, Pieter Noordhuis", + author_email="janerik@fnordig.de, pcnoordhuis@gmail.com", + keywords=["Redis"], + license="BSD", + packages=["hiredis"], + package_data={"hiredis": ["hiredis.pyi", "py.typed"]}, + ext_modules=[ext], + python_requires=">=3.7", + project_urls={ "Changes": "https://github.com/redis/hiredis-py/releases", "Issue tracker": "https://github.com/redis/hiredis-py/issues", - }, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: MacOS', - 'Operating System :: POSIX', - 'Programming Language :: C', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: Software Development', - ], + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: MacOS', + 'Operating System :: POSIX', + 'Programming Language :: C', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Software Development', + ], ) From 5807460a23916e838e1d82ab6dfdfb821d288075 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 25 Jan 2023 09:55:08 -0600 Subject: [PATCH 12/21] Drop extra link args --- hiredis.egg-info/PKG-INFO | 185 ++++++++++++++++++++++++++ hiredis.egg-info/SOURCES.txt | 43 ++++++ hiredis.egg-info/dependency_links.txt | 1 + hiredis.egg-info/top_level.txt | 1 + setup.py | 1 - 5 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 hiredis.egg-info/PKG-INFO create mode 100644 hiredis.egg-info/SOURCES.txt create mode 100644 hiredis.egg-info/dependency_links.txt create mode 100644 hiredis.egg-info/top_level.txt diff --git a/hiredis.egg-info/PKG-INFO b/hiredis.egg-info/PKG-INFO new file mode 100644 index 00000000..ff99434b --- /dev/null +++ b/hiredis.egg-info/PKG-INFO @@ -0,0 +1,185 @@ +Metadata-Version: 2.1 +Name: hiredis +Version: 2.1.1 +Summary: Python wrapper for hiredis +Home-page: https://github.com/redis/hiredis-py +Author: Jan-Erik Rediger, Pieter Noordhuis +Author-email: janerik@fnordig.de, pcnoordhuis@gmail.com +License: BSD +Project-URL: Changes, https://github.com/redis/hiredis-py/releases +Project-URL: Issue tracker, https://github.com/redis/hiredis-py/issues +Keywords: Redis +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: MacOS +Classifier: Operating System :: POSIX +Classifier: Programming Language :: C +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: Software Development +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +License-File: LICENSE + +# hiredis-py + +[![Build Status](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml/badge.svg)](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml) +[![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Python extension that wraps protocol parsing code in [hiredis][hiredis]. +It primarily speeds up parsing of multi bulk replies. + +[hiredis]: http://github.com/redis/hiredis + +## Install + +hiredis-py is available on [PyPI](https://pypi.org/project/hiredis/), and can be installed via: + +```bash +pip install hiredis +``` +## Building and Testing + +Building this repository requires a recursive checkout of submodules, and building hiredis. The following example shows how to clone, compile, and run tests. Please note - you will need the gcc installed. + +```bash +git clone --recursse-submodules https://github.com/redis/hiredis-py +python setup.py build_ext --inplace +pytest +``` + +### Requirements + +hiredis-py requires **Python 3.7+**. + +Make sure Python development headers are available when installing hiredis-py. +On Ubuntu/Debian systems, install them with `apt-get install python3-dev`. + +## Usage + +The `hiredis` module contains the `Reader` class. This class is responsible for +parsing replies from the stream of data that is read from a Redis connection. +It does not contain functionality to handle I/O. + +### Reply parser + +The `Reader` class has two methods that are used when parsing replies from a +stream of data. `Reader.feed` takes a string argument that is appended to the +internal buffer. `Reader.gets` reads this buffer and returns a reply when the +buffer contains a full reply. If a single call to `feed` contains multiple +replies, `gets` should be called multiple times to extract all replies. + +Example: + +```python +>>> reader = hiredis.Reader() +>>> reader.feed("$5\r\nhello\r\n") +>>> reader.gets() +b'hello' +``` + +When the buffer does not contain a full reply, `gets` returns `False`. +This means extra data is needed and `feed` should be called again before calling +`gets` again. Alternatively you could provide custom sentinel object via parameter, +which is useful for RESP3 protocol where native boolean types are supported: + +Example: + +```python +>>> reader.feed("*2\r\n$5\r\nhello\r\n") +>>> reader.gets() +False +>>> reader.feed("$5\r\nworld\r\n") +>>> reader.gets() +[b'hello', b'world'] +>>> reader = hiredis.Reader(notEnoughData=Ellipsis) +>>> reader.gets() +Ellipsis +``` + +#### Unicode + +`hiredis.Reader` is able to decode bulk data to any encoding Python supports. +To do so, specify the encoding you want to use for decoding replies when +initializing it: + +```python +>>> reader = hiredis.Reader(encoding="utf-8", errors="strict") +>>> reader.feed(b"$3\r\n\xe2\x98\x83\r\n") +>>> reader.gets() +'☃' +``` + +Decoding of bulk data will be attempted using the specified encoding and +error handler. If the error handler is `'strict'` (the default), a +`UnicodeDecodeError` is raised when data cannot be dedcoded. This is identical +to Python's default behavior. Other valid values to `errors` include +`'replace'`, `'ignore'`, and `'backslashreplace'`. More information on the +behavior of these error handlers can be found +[here](https://docs.python.org/3/howto/unicode.html#the-string-type). + + +When the specified encoding cannot be found, a `LookupError` will be raised +when calling `gets` for the first reply with bulk data. + +#### Error handling + +When a protocol error occurs (because of multiple threads using the same +socket, or some other condition that causes a corrupt stream), the error +`hiredis.ProtocolError` is raised. Because the buffer is read in a lazy +fashion, it will only be raised when `gets` is called and the first reply in +the buffer contains an error. There is no way to recover from a faulty protocol +state, so when this happens, the I/O code feeding data to `Reader` should +probably reconnect. + +Redis can reply with error replies (`-ERR ...`). For these replies, the custom +error class `hiredis.ReplyError` is returned, **but not raised**. + +When other error types should be used (so existing code doesn't have to change +its `except` clauses), `Reader` can be initialized with the `protocolError` and +`replyError` keywords. These keywords should contain a *class* that is a +subclass of `Exception`. When not provided, `Reader` will use the default +error types. + +## Benchmarks + +The repository contains a benchmarking script in the `benchmark` directory, +which uses [gevent](http://gevent.org/) to have non-blocking I/O and redis-py +to handle connections. These benchmarks are done with a patched version of +redis-py that uses hiredis-py when it is available. + +All benchmarks are done with 10 concurrent connections. + +* SET key value + GET key + * redis-py: 11.76 Kops + * redis-py *with* hiredis-py: 13.40 Kops + * improvement: **1.1x** + +List entries in the following tests are 5 bytes. + +* LRANGE list 0 **9**: + * redis-py: 4.78 Kops + * redis-py *with* hiredis-py: 12.94 Kops + * improvement: **2.7x** +* LRANGE list 0 **99**: + * redis-py: 0.73 Kops + * redis-py *with* hiredis-py: 11.90 Kops + * improvement: **16.3x** +* LRANGE list 0 **999**: + * redis-py: 0.07 Kops + * redis-py *with* hiredis-py: 5.83 Kops + * improvement: **83.2x** + +Throughput improvement for simple SET/GET is minimal, but the larger multi bulk replies +get, the larger the performance improvement is. + +## License + +This code is released under the BSD license, after the license of hiredis. diff --git a/hiredis.egg-info/SOURCES.txt b/hiredis.egg-info/SOURCES.txt new file mode 100644 index 00000000..c2e20646 --- /dev/null +++ b/hiredis.egg-info/SOURCES.txt @@ -0,0 +1,43 @@ +LICENSE +MANIFEST.in +README.md +setup.cfg +setup.py +hiredis/__init__.py +hiredis/hiredis.pyi +hiredis/py.typed +hiredis/version.py +hiredis.egg-info/PKG-INFO +hiredis.egg-info/SOURCES.txt +hiredis.egg-info/dependency_links.txt +hiredis.egg-info/top_level.txt +src/hiredis.c +src/hiredis.h +src/pack.c +src/pack.h +src/reader.c +src/reader.h +vendor/hiredis/COPYING +vendor/hiredis/alloc.c +vendor/hiredis/alloc.h +vendor/hiredis/async.c +vendor/hiredis/async.h +vendor/hiredis/async_private.h +vendor/hiredis/dict.c +vendor/hiredis/dict.h +vendor/hiredis/fmacros.h +vendor/hiredis/hiredis.c +vendor/hiredis/hiredis.h +vendor/hiredis/hiredis_ssl.h +vendor/hiredis/net.c +vendor/hiredis/net.h +vendor/hiredis/read.c +vendor/hiredis/read.h +vendor/hiredis/sds.c +vendor/hiredis/sds.h +vendor/hiredis/sdsalloc.h +vendor/hiredis/sockcompat.c +vendor/hiredis/sockcompat.h +vendor/hiredis/ssl.c +vendor/hiredis/test.c +vendor/hiredis/win32.h \ No newline at end of file diff --git a/hiredis.egg-info/dependency_links.txt b/hiredis.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/hiredis.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/hiredis.egg-info/top_level.txt b/hiredis.egg-info/top_level.txt new file mode 100644 index 00000000..01e78051 --- /dev/null +++ b/hiredis.egg-info/top_level.txt @@ -0,0 +1 @@ +hiredis diff --git a/setup.py b/setup.py index 27319752..0db32200 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ def version(): sources=sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), extra_compile_args=["-std=c99"], - extra_link_args=["-Wl,-Bsymbolic"], include_dirs=["vendor"]) setup( From 8643b7c49bf08757e100c35594b0ea2ec9f8682e Mon Sep 17 00:00:00 2001 From: zalmane Date: Wed, 25 Jan 2023 19:33:27 +0200 Subject: [PATCH 13/21] add conditional compile flags excluding windows and mac --- setup.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 0db32200..9797b374 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,32 @@ #!/usr/bin/env python try: - from setuptools import setup, Extension + from setuptools import setup, Extension except ImportError: - from distutils.core import setup, Extension + from distutils.core import setup, Extension import importlib import glob import io +import sys def version(): - loader = importlib.machinery.SourceFileLoader("hiredis.version", "hiredis/version.py") - module = loader.load_module() - return module.__version__ + loader = importlib.machinery.SourceFileLoader("hiredis.version", "hiredis/version.py") + module = loader.load_module() + return module.__version__ -ext = Extension("hiredis.hiredis", - sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), - extra_compile_args=["-std=c99"], - include_dirs=["vendor"]) +if 'win' in sys.platform or 'darwin' in sys.platform: + extra_compile_args = [] +else: + extra_compile_args = ["-std=c99"] + +ext = Extension( + "hiredis.hiredis", + sources=sorted( + glob.glob("src/*.c") + + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + extra_compile_args=extra_compile_args, include_dirs=["vendor"]) setup( name="hiredis", From 3517354b684b37f20b4d457ee0ad399481e162dc Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 25 Jan 2023 11:35:34 -0600 Subject: [PATCH 14/21] Remove egg-info --- hiredis.egg-info/PKG-INFO | 185 -------------------------- hiredis.egg-info/SOURCES.txt | 43 ------ hiredis.egg-info/dependency_links.txt | 1 - hiredis.egg-info/top_level.txt | 1 - 4 files changed, 230 deletions(-) delete mode 100644 hiredis.egg-info/PKG-INFO delete mode 100644 hiredis.egg-info/SOURCES.txt delete mode 100644 hiredis.egg-info/dependency_links.txt delete mode 100644 hiredis.egg-info/top_level.txt diff --git a/hiredis.egg-info/PKG-INFO b/hiredis.egg-info/PKG-INFO deleted file mode 100644 index ff99434b..00000000 --- a/hiredis.egg-info/PKG-INFO +++ /dev/null @@ -1,185 +0,0 @@ -Metadata-Version: 2.1 -Name: hiredis -Version: 2.1.1 -Summary: Python wrapper for hiredis -Home-page: https://github.com/redis/hiredis-py -Author: Jan-Erik Rediger, Pieter Noordhuis -Author-email: janerik@fnordig.de, pcnoordhuis@gmail.com -License: BSD -Project-URL: Changes, https://github.com/redis/hiredis-py/releases -Project-URL: Issue tracker, https://github.com/redis/hiredis-py/issues -Keywords: Redis -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: MacOS -Classifier: Operating System :: POSIX -Classifier: Programming Language :: C -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Topic :: Software Development -Requires-Python: >=3.7 -Description-Content-Type: text/markdown -License-File: LICENSE - -# hiredis-py - -[![Build Status](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml/badge.svg)](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml) -[![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) - -Python extension that wraps protocol parsing code in [hiredis][hiredis]. -It primarily speeds up parsing of multi bulk replies. - -[hiredis]: http://github.com/redis/hiredis - -## Install - -hiredis-py is available on [PyPI](https://pypi.org/project/hiredis/), and can be installed via: - -```bash -pip install hiredis -``` -## Building and Testing - -Building this repository requires a recursive checkout of submodules, and building hiredis. The following example shows how to clone, compile, and run tests. Please note - you will need the gcc installed. - -```bash -git clone --recursse-submodules https://github.com/redis/hiredis-py -python setup.py build_ext --inplace -pytest -``` - -### Requirements - -hiredis-py requires **Python 3.7+**. - -Make sure Python development headers are available when installing hiredis-py. -On Ubuntu/Debian systems, install them with `apt-get install python3-dev`. - -## Usage - -The `hiredis` module contains the `Reader` class. This class is responsible for -parsing replies from the stream of data that is read from a Redis connection. -It does not contain functionality to handle I/O. - -### Reply parser - -The `Reader` class has two methods that are used when parsing replies from a -stream of data. `Reader.feed` takes a string argument that is appended to the -internal buffer. `Reader.gets` reads this buffer and returns a reply when the -buffer contains a full reply. If a single call to `feed` contains multiple -replies, `gets` should be called multiple times to extract all replies. - -Example: - -```python ->>> reader = hiredis.Reader() ->>> reader.feed("$5\r\nhello\r\n") ->>> reader.gets() -b'hello' -``` - -When the buffer does not contain a full reply, `gets` returns `False`. -This means extra data is needed and `feed` should be called again before calling -`gets` again. Alternatively you could provide custom sentinel object via parameter, -which is useful for RESP3 protocol where native boolean types are supported: - -Example: - -```python ->>> reader.feed("*2\r\n$5\r\nhello\r\n") ->>> reader.gets() -False ->>> reader.feed("$5\r\nworld\r\n") ->>> reader.gets() -[b'hello', b'world'] ->>> reader = hiredis.Reader(notEnoughData=Ellipsis) ->>> reader.gets() -Ellipsis -``` - -#### Unicode - -`hiredis.Reader` is able to decode bulk data to any encoding Python supports. -To do so, specify the encoding you want to use for decoding replies when -initializing it: - -```python ->>> reader = hiredis.Reader(encoding="utf-8", errors="strict") ->>> reader.feed(b"$3\r\n\xe2\x98\x83\r\n") ->>> reader.gets() -'☃' -``` - -Decoding of bulk data will be attempted using the specified encoding and -error handler. If the error handler is `'strict'` (the default), a -`UnicodeDecodeError` is raised when data cannot be dedcoded. This is identical -to Python's default behavior. Other valid values to `errors` include -`'replace'`, `'ignore'`, and `'backslashreplace'`. More information on the -behavior of these error handlers can be found -[here](https://docs.python.org/3/howto/unicode.html#the-string-type). - - -When the specified encoding cannot be found, a `LookupError` will be raised -when calling `gets` for the first reply with bulk data. - -#### Error handling - -When a protocol error occurs (because of multiple threads using the same -socket, or some other condition that causes a corrupt stream), the error -`hiredis.ProtocolError` is raised. Because the buffer is read in a lazy -fashion, it will only be raised when `gets` is called and the first reply in -the buffer contains an error. There is no way to recover from a faulty protocol -state, so when this happens, the I/O code feeding data to `Reader` should -probably reconnect. - -Redis can reply with error replies (`-ERR ...`). For these replies, the custom -error class `hiredis.ReplyError` is returned, **but not raised**. - -When other error types should be used (so existing code doesn't have to change -its `except` clauses), `Reader` can be initialized with the `protocolError` and -`replyError` keywords. These keywords should contain a *class* that is a -subclass of `Exception`. When not provided, `Reader` will use the default -error types. - -## Benchmarks - -The repository contains a benchmarking script in the `benchmark` directory, -which uses [gevent](http://gevent.org/) to have non-blocking I/O and redis-py -to handle connections. These benchmarks are done with a patched version of -redis-py that uses hiredis-py when it is available. - -All benchmarks are done with 10 concurrent connections. - -* SET key value + GET key - * redis-py: 11.76 Kops - * redis-py *with* hiredis-py: 13.40 Kops - * improvement: **1.1x** - -List entries in the following tests are 5 bytes. - -* LRANGE list 0 **9**: - * redis-py: 4.78 Kops - * redis-py *with* hiredis-py: 12.94 Kops - * improvement: **2.7x** -* LRANGE list 0 **99**: - * redis-py: 0.73 Kops - * redis-py *with* hiredis-py: 11.90 Kops - * improvement: **16.3x** -* LRANGE list 0 **999**: - * redis-py: 0.07 Kops - * redis-py *with* hiredis-py: 5.83 Kops - * improvement: **83.2x** - -Throughput improvement for simple SET/GET is minimal, but the larger multi bulk replies -get, the larger the performance improvement is. - -## License - -This code is released under the BSD license, after the license of hiredis. diff --git a/hiredis.egg-info/SOURCES.txt b/hiredis.egg-info/SOURCES.txt deleted file mode 100644 index c2e20646..00000000 --- a/hiredis.egg-info/SOURCES.txt +++ /dev/null @@ -1,43 +0,0 @@ -LICENSE -MANIFEST.in -README.md -setup.cfg -setup.py -hiredis/__init__.py -hiredis/hiredis.pyi -hiredis/py.typed -hiredis/version.py -hiredis.egg-info/PKG-INFO -hiredis.egg-info/SOURCES.txt -hiredis.egg-info/dependency_links.txt -hiredis.egg-info/top_level.txt -src/hiredis.c -src/hiredis.h -src/pack.c -src/pack.h -src/reader.c -src/reader.h -vendor/hiredis/COPYING -vendor/hiredis/alloc.c -vendor/hiredis/alloc.h -vendor/hiredis/async.c -vendor/hiredis/async.h -vendor/hiredis/async_private.h -vendor/hiredis/dict.c -vendor/hiredis/dict.h -vendor/hiredis/fmacros.h -vendor/hiredis/hiredis.c -vendor/hiredis/hiredis.h -vendor/hiredis/hiredis_ssl.h -vendor/hiredis/net.c -vendor/hiredis/net.h -vendor/hiredis/read.c -vendor/hiredis/read.h -vendor/hiredis/sds.c -vendor/hiredis/sds.h -vendor/hiredis/sdsalloc.h -vendor/hiredis/sockcompat.c -vendor/hiredis/sockcompat.h -vendor/hiredis/ssl.c -vendor/hiredis/test.c -vendor/hiredis/win32.h \ No newline at end of file diff --git a/hiredis.egg-info/dependency_links.txt b/hiredis.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/hiredis.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/hiredis.egg-info/top_level.txt b/hiredis.egg-info/top_level.txt deleted file mode 100644 index 01e78051..00000000 --- a/hiredis.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -hiredis From 05fe090b6a811a0388bea92dc021ad4de036f47c Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 25 Jan 2023 12:03:19 -0600 Subject: [PATCH 15/21] 1) add conditional compile flags excluding windows and mac 2) add egg-info to gitignore --- .gitignore | 1 + setup.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index dbe349b2..86cb2c86 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ MANIFEST .venv **/*.so +hiredis.egg-info diff --git a/setup.py b/setup.py index 9797b374..2dc31dc3 100644 --- a/setup.py +++ b/setup.py @@ -11,22 +11,23 @@ def version(): - loader = importlib.machinery.SourceFileLoader("hiredis.version", "hiredis/version.py") + loader = importlib.machinery.SourceFileLoader( + "hiredis.version", "hiredis/version.py") module = loader.load_module() return module.__version__ if 'win' in sys.platform or 'darwin' in sys.platform: - extra_compile_args = [] + extra_link_args = [] else: - extra_compile_args = ["-std=c99"] + extra_link_args = ["-Wl,-Bsymbolic"] -ext = Extension( - "hiredis.hiredis", - sources=sorted( - glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), - extra_compile_args=extra_compile_args, include_dirs=["vendor"]) +ext = Extension("hiredis.hiredis", + sources=sorted(glob.glob("src/*.c") + + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + extra_compile_args=["-std=c99"], + extra_link_args=["-Wl,-Bsymbolic"], + include_dirs=["vendor"]) setup( name="hiredis", From 2ffa3623733deeabb71f655900fe0409c68d7598 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 25 Jan 2023 12:12:02 -0600 Subject: [PATCH 16/21] Actually use conditional compile flags --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 2dc31dc3..f01a1e64 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def version(): sources=sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), extra_compile_args=["-std=c99"], - extra_link_args=["-Wl,-Bsymbolic"], + extra_link_args=extra_link_args, include_dirs=["vendor"]) setup( From 1758fe9e76269383a1e370f010fc9e9709598cf3 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 26 Jan 2023 10:34:39 -0600 Subject: [PATCH 17/21] Fix the Windows build. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index f01a1e64..43732a29 --- a/setup.py +++ b/setup.py @@ -22,9 +22,11 @@ def version(): else: extra_link_args = ["-Wl,-Bsymbolic"] +hiredis_files = ("alloc", "async", "hiredis", "net", "read", "sds", "sockcompat") + ext = Extension("hiredis.hiredis", sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in ("alloc", "async", "hiredis", "net", "read", "sds")]), + ["vendor/hiredis/%s.c" % src for src in hiredis_files]), extra_compile_args=["-std=c99"], extra_link_args=extra_link_args, include_dirs=["vendor"]) From 11fa2354d408d691a172ee1587c26bd7cc849c9f Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 26 Jan 2023 12:59:13 -0600 Subject: [PATCH 18/21] Re-factored setup.py and actually fixed the Windows build. --- setup.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 43732a29..67d3fbac 100644 --- a/setup.py +++ b/setup.py @@ -16,19 +16,34 @@ def version(): module = loader.load_module() return module.__version__ +def get_sources(): + hiredis_sources = ("alloc", "async", "hiredis", "net", "read", "sds", "sockcompat") + return sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in hiredis_sources]) -if 'win' in sys.platform or 'darwin' in sys.platform: - extra_link_args = [] -else: - extra_link_args = ["-Wl,-Bsymbolic"] +def get_linker_args(): + if 'win' in sys.platform or 'darwin' in sys.platform: + return [] + else: + return ["-Wl,-Bsymbolic"] + +def get_compiler_args(): + if 'win' in sys.platform: + return [] + else: + return ["-std=c99",] + +def get_libraries(): + if 'win' in sys.platform: + return ["ws2_32",] + else: + return [] -hiredis_files = ("alloc", "async", "hiredis", "net", "read", "sds", "sockcompat") ext = Extension("hiredis.hiredis", - sources=sorted(glob.glob("src/*.c") + - ["vendor/hiredis/%s.c" % src for src in hiredis_files]), - extra_compile_args=["-std=c99"], - extra_link_args=extra_link_args, + sources=get_sources(), + extra_compile_args=get_compiler_args(), + extra_link_args=get_linker_args(), + libraries=get_libraries(), include_dirs=["vendor"]) setup( From d1a5ebee297168cb38fa63603acf3c0ce97541fd Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 26 Jan 2023 14:28:20 -0600 Subject: [PATCH 19/21] Apparently 'win' is not the same as win32. --- setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 67d3fbac..c506c17c 100644 --- a/setup.py +++ b/setup.py @@ -16,24 +16,28 @@ def version(): module = loader.load_module() return module.__version__ + def get_sources(): hiredis_sources = ("alloc", "async", "hiredis", "net", "read", "sds", "sockcompat") return sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in hiredis_sources]) + def get_linker_args(): - if 'win' in sys.platform or 'darwin' in sys.platform: + if 'win32' in sys.platform or 'darwin' in sys.platform: return [] else: - return ["-Wl,-Bsymbolic"] + return ["-Wl,-Bsymbolic",] + def get_compiler_args(): - if 'win' in sys.platform: + if 'win32' in sys.platform: return [] else: return ["-std=c99",] + def get_libraries(): - if 'win' in sys.platform: + if 'win32' in sys.platform: return ["ws2_32",] else: return [] From 1f1af09b7d1dc918dab3b240f55e694881ed32f0 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Tue, 31 Jan 2023 09:34:56 -0600 Subject: [PATCH 20/21] 1) Don't use Bsymbolic linker arg 2) Fix a build break on x86 platforms --- setup.py | 10 +--------- src/pack.c | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index c506c17c..7e708077 100644 --- a/setup.py +++ b/setup.py @@ -22,18 +22,11 @@ def get_sources(): return sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in hiredis_sources]) -def get_linker_args(): - if 'win32' in sys.platform or 'darwin' in sys.platform: - return [] - else: - return ["-Wl,-Bsymbolic",] - - def get_compiler_args(): if 'win32' in sys.platform: return [] else: - return ["-std=c99",] + return ["-std=c99", "-static-libstdc++", "-O2"] def get_libraries(): @@ -46,7 +39,6 @@ def get_libraries(): ext = Extension("hiredis.hiredis", sources=get_sources(), extra_compile_args=get_compiler_args(), - extra_link_args=get_linker_args(), libraries=get_libraries(), include_dirs=["vendor"]) diff --git a/src/pack.c b/src/pack.c index 0272a26f..f9b6724a 100644 --- a/src/pack.c +++ b/src/pack.c @@ -1,5 +1,21 @@ #include "pack.h" + +#ifndef _MSC_VER #include +#else +/* Workaround for https://bugs.python.org/issue11717. + * defines ssize_t which can conflict + * with Python's definition. + */ +extern long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); +typedef char *sds; +extern void sds_free(void *ptr); +extern sds sdsempty(void); +extern void sdsfreesplitres(sds *tokens, int count); +extern sds sdscpylen(sds s, const char *t, size_t len); +extern sds sdsnewlen(const void *init, size_t initlen); +#endif + #include PyObject * @@ -15,13 +31,13 @@ pack_command(PyObject *cmd) return NULL; } - int tokens_number = PyTuple_Size(cmd); + Py_ssize_t tokens_number = PyTuple_Size(cmd); sds *tokens = s_malloc(sizeof(sds) * tokens_number); if (tokens == NULL) { return PyErr_NoMemory(); } - + memset(tokens, 0, sizeof(sds) * tokens_number); size_t *lengths = hi_malloc(sizeof(size_t) * tokens_number); @@ -32,7 +48,7 @@ pack_command(PyObject *cmd) } Py_ssize_t len = 0; - + for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) { PyObject *item = PyTuple_GetItem(cmd, i); From 29050282b4b13df840d05b150278c90f04dbfad7 Mon Sep 17 00:00:00 2001 From: zalmane Date: Thu, 9 Feb 2023 14:51:32 +0200 Subject: [PATCH 21/21] revert back to using Bsymbolic --- setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7e708077..905df590 100644 --- a/setup.py +++ b/setup.py @@ -22,16 +22,23 @@ def get_sources(): return sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in hiredis_sources]) +def get_linker_args(): + if 'win32' in sys.platform or 'darwin' in sys.platform: + return [] + else: + return ["-Wl,-Bsymbolic", ] + + def get_compiler_args(): if 'win32' in sys.platform: return [] else: - return ["-std=c99", "-static-libstdc++", "-O2"] + return ["-std=c99"] def get_libraries(): if 'win32' in sys.platform: - return ["ws2_32",] + return ["ws2_32", ] else: return [] @@ -39,6 +46,7 @@ def get_libraries(): ext = Extension("hiredis.hiredis", sources=get_sources(), extra_compile_args=get_compiler_args(), + extra_link_args=get_linker_args(), libraries=get_libraries(), include_dirs=["vendor"])