From cba777a1e14f836028e408d83d1b099748c75c93 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 11 Dec 2020 12:00:10 +0000 Subject: [PATCH 01/14] allow avoiding try/catch in with_gil --- src/gil.jl | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/gil.jl b/src/gil.jl index 01785440..563a058d 100644 --- a/src/gil.jl +++ b/src/gil.jl @@ -1,5 +1,14 @@ -function with_gil(f) - if CONFIG.isembedded +""" + with_gil(f, [c=true]) + +Compute `f()` with the GIL enabled. + +This may need a `try-finally` block to ensure the GIL is released again. If you know that `f` cannot throw, pass `c=false` to avoid this overhead. +""" +function with_gil(f, c::Bool=true) + if !CONFIG.isembedded + f() + elseif c g = C.PyGILState_Ensure() try f() @@ -7,6 +16,9 @@ function with_gil(f) C.PyGILState_Release(g) end else - f() + g = C.PyGILState_Ensure() + r = f() + C.PyGILState_Release(g) + r end end From 13c0791cdcc5a9d74da10e2777b1f800f3644c21 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 11 Dec 2020 12:03:40 +0000 Subject: [PATCH 02/14] avoid try/catch in finalizers --- src/PyBuffer.jl | 2 +- src/PyObjectArray.jl | 2 +- src/object.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PyBuffer.jl b/src/PyBuffer.jl index 3fae550e..68dbb11b 100644 --- a/src/PyBuffer.jl +++ b/src/PyBuffer.jl @@ -26,7 +26,7 @@ mutable struct PyBuffer b = new(info) finalizer(b) do b if CONFIG.isinitialized - err = with_gil() do + err = with_gil(false) do C.PyBuffer_Release(pointer(b.info)) end check(err) diff --git a/src/PyObjectArray.jl b/src/PyObjectArray.jl index 3365ab2d..9a5f62d7 100644 --- a/src/PyObjectArray.jl +++ b/src/PyObjectArray.jl @@ -4,7 +4,7 @@ mutable struct PyObjectArray{N} <: AbstractArray{PyObject, N} x = new{N}(fill(CPyPtr(C_NULL), dims)) finalizer(x) do x if CONFIG.isinitialized - with_gil() do + with_gil(false) do for ptr in x.ptrs C.Py_DecRef(ptr) end diff --git a/src/object.jl b/src/object.jl index 3127b240..d5a803a9 100644 --- a/src/object.jl +++ b/src/object.jl @@ -14,7 +14,7 @@ mutable struct PyObject if CONFIG.isinitialized ptr = getfield(o, :ptr) if ptr != C_NULL - with_gil() do + with_gil(false) do C.Py_DecRef(ptr) end end From 44869019317f3da8dc2743c4f982cd072e5b524b Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sun, 13 Dec 2020 17:49:02 +0000 Subject: [PATCH 03/14] faster conversion --- src/PyCode.jl | 37 ++ src/PyDict.jl | 86 ++-- src/PyException.jl | 153 ++++++ src/Python.jl | 152 +++--- src/cpython.jl | 511 -------------------- src/cpython/CPython.jl | 833 +++++++++++++++++++++++++++++++++ src/eval.jl | 212 ++++----- src/init.jl | 157 ++++--- src/{ => old}/PyArray.jl | 0 src/{ => old}/PyBuffer.jl | 0 src/old/PyDict.jl | 61 +++ src/{ => old}/PyIO.jl | 0 src/{ => old}/PyIterable.jl | 0 src/{ => old}/PyList.jl | 0 src/{ => old}/PyObjectArray.jl | 0 src/{ => old}/PySet.jl | 0 src/{ => old}/base.jl | 0 src/{ => old}/bool.jl | 0 src/{ => old}/builtins.jl | 0 src/{ => old}/bytearray.jl | 0 src/{ => old}/bytes.jl | 0 src/{ => old}/collections.jl | 0 src/{ => old}/complex.jl | 0 src/{ => old}/convert.jl | 0 src/{ => old}/datetime.jl | 0 src/{ => old}/dict.jl | 0 src/{ => old}/error.jl | 0 src/old/eval.jl | 137 ++++++ src/{ => old}/float.jl | 0 src/{ => old}/fraction.jl | 0 src/{ => old}/function.jl | 0 src/{ => old}/gui.jl | 0 src/{ => old}/import.jl | 0 src/{ => old}/int.jl | 0 src/{ => old}/io.jl | 0 src/{ => old}/julia.jl | 0 src/{ => old}/list.jl | 0 src/{ => old}/matplotlib.jl | 0 src/{ => old}/newtype.jl | 0 src/{ => old}/none.jl | 0 src/{ => old}/number.jl | 0 src/{ => old}/numpy.jl | 0 src/{ => old}/object.jl | 0 src/{ => old}/pandas.jl | 0 src/{ => old}/pywith.jl | 0 src/{ => old}/range.jl | 0 src/{ => old}/sequence.jl | 0 src/{ => old}/set.jl | 0 src/{ => old}/slice.jl | 0 src/{ => old}/stdlib.jl | 0 src/{ => old}/str.jl | 0 src/{ => old}/tuple.jl | 0 src/{ => old}/type.jl | 0 src/utils.jl | 4 + 54 files changed, 1500 insertions(+), 843 deletions(-) create mode 100644 src/PyCode.jl create mode 100644 src/PyException.jl delete mode 100644 src/cpython.jl create mode 100644 src/cpython/CPython.jl rename src/{ => old}/PyArray.jl (100%) rename src/{ => old}/PyBuffer.jl (100%) create mode 100644 src/old/PyDict.jl rename src/{ => old}/PyIO.jl (100%) rename src/{ => old}/PyIterable.jl (100%) rename src/{ => old}/PyList.jl (100%) rename src/{ => old}/PyObjectArray.jl (100%) rename src/{ => old}/PySet.jl (100%) rename src/{ => old}/base.jl (100%) rename src/{ => old}/bool.jl (100%) rename src/{ => old}/builtins.jl (100%) rename src/{ => old}/bytearray.jl (100%) rename src/{ => old}/bytes.jl (100%) rename src/{ => old}/collections.jl (100%) rename src/{ => old}/complex.jl (100%) rename src/{ => old}/convert.jl (100%) rename src/{ => old}/datetime.jl (100%) rename src/{ => old}/dict.jl (100%) rename src/{ => old}/error.jl (100%) create mode 100644 src/old/eval.jl rename src/{ => old}/float.jl (100%) rename src/{ => old}/fraction.jl (100%) rename src/{ => old}/function.jl (100%) rename src/{ => old}/gui.jl (100%) rename src/{ => old}/import.jl (100%) rename src/{ => old}/int.jl (100%) rename src/{ => old}/io.jl (100%) rename src/{ => old}/julia.jl (100%) rename src/{ => old}/list.jl (100%) rename src/{ => old}/matplotlib.jl (100%) rename src/{ => old}/newtype.jl (100%) rename src/{ => old}/none.jl (100%) rename src/{ => old}/number.jl (100%) rename src/{ => old}/numpy.jl (100%) rename src/{ => old}/object.jl (100%) rename src/{ => old}/pandas.jl (100%) rename src/{ => old}/pywith.jl (100%) rename src/{ => old}/range.jl (100%) rename src/{ => old}/sequence.jl (100%) rename src/{ => old}/set.jl (100%) rename src/{ => old}/slice.jl (100%) rename src/{ => old}/stdlib.jl (100%) rename src/{ => old}/str.jl (100%) rename src/{ => old}/tuple.jl (100%) rename src/{ => old}/type.jl (100%) diff --git a/src/PyCode.jl b/src/PyCode.jl new file mode 100644 index 00000000..705bc7ad --- /dev/null +++ b/src/PyCode.jl @@ -0,0 +1,37 @@ +mutable struct PyCode + ptr :: CPyPtr + code :: String + filename :: String + mode :: Symbol + function PyCode(code::String, filename::String, mode::Symbol) + mode in (:exec, :eval) || error("invalid mode $(repr(mode))") + o = new(CPyPtr(), code, filename, mode) + finalizer(o) do o + if CONFIG.isinitialized + ptr = getfield(o, :ptr) + if !isnull(ptr) + with_gil(false) do + C.Py_DecRef(ptr) + end + end + end + end + return o + end +end +export PyCode + +function pyptr(co::PyCode) + ptr = getfield(co, :ptr) + if isnull(ptr) + ptr = C.Py_CompileString(co.code, co.filename, co.mode == :exec ? C.Py_file_input : co.mode == :eval ? C.Py_eval_input : error("invalid mode $(repr(co.mode))")) + if isnull(ptr) + pythrow() + else + setfield!(co, :ptr, ptr) + ptr + end + else + ptr + end +end diff --git a/src/PyDict.jl b/src/PyDict.jl index 370d882d..d02de681 100644 --- a/src/PyDict.jl +++ b/src/PyDict.jl @@ -1,61 +1,37 @@ -""" - PyDict{K=PyObject, V=PyObject}(o=pydict()) - -Wrap the Python dictionary `o` (or anything satisfying the mapping interface) as a Julia dictionary with keys of type `K` and values of type `V`. -""" -struct PyDict{K,V} <: AbstractDict{K,V} - o :: PyObject - PyDict{K,V}(o::PyObject) where {K,V} = new{K,V}(o) +mutable struct PyDict + ptr :: CPyPtr + hasbuiltins :: Bool + function PyDict() + o = new(CPyPtr(), false) + finalizer(o) do o + if CONFIG.isinitialized + ptr = getfield(o, :ptr) + if !isnull(ptr) + with_gil(false) do + C.Py_DecRef(ptr) + end + end + end + end + return o + end end -PyDict{K,V}(o=pydict()) where {K,V} = PyDict{K,V}(pydict(o)) -PyDict{K}(o=pydict()) where {K} = PyDict{K,PyObject}(o) -PyDict(o=pydict()) = PyDict{PyObject}(o) export PyDict -pyobject(x::PyDict) = x.o - -function Base.iterate(x::PyDict{K,V}, it=pyiter(x.o.items())) where {K,V} - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing +const pyglobals = PyDict() +export pyglobals + +function pyptr(x::PyDict) + ptr = getfield(x, :ptr) + if isnull(ptr) + ptr = C.PyDict_New() + if isnull(ptr) + pythrow() + else + setfield!(x, :ptr, ptr) + ptr + end else - kv = pynewobject(ptr) - (pyconvert(K, kv[0]) => pyconvert(V, kv[1])), it + ptr end end - -Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = - (pysetitem(x.o, convert(K, k), convert(V, v)); x) - -Base.getindex(x::PyDict{K,V}, k) where {K,V} = - pyconvert(V, pygetitem(x.o, convert(K, k))) - -Base.delete!(x::PyDict{K,V}, k) where {K,V} = - (pydelitem(x.o, convert(K, k)); x) - -Base.length(x::PyDict) = Int(pylen(x.o)) - -Base.empty!(x::PyDict) = (x.o.clear(); x) - -Base.copy(x::PyDict) = typeof(x)(x.o.copy()) - -function Base.get(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : d -end - -function Base.get(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : f() -end - -function Base.get!(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = d) -end - -function Base.get!(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = f()) -end diff --git a/src/PyException.jl b/src/PyException.jl new file mode 100644 index 00000000..5f7fc18a --- /dev/null +++ b/src/PyException.jl @@ -0,0 +1,153 @@ +mutable struct PyException <: Exception + tptr :: CPyPtr + vptr :: CPyPtr + bptr :: CPyPtr + function PyException(::Val{:new}, t::Ptr=C_NULL, v::Ptr=C_NULL, b::Ptr=C_NULL, borrowed::Bool=false) + o = new(CPyPtr(t), CPyPtr(v), CPyPtr(b)) + if borrowed + C.Py_IncRef(t) + C.Py_IncRef(v) + C.Py_IncRef(b) + end + finalizer(o) do o + if CONFIG.isinitialized + tptr = getfield(o, :tptr) + vptr = getfield(o, :vptr) + bptr = getfield(o, :bptr) + if !isnull(tptr) || !isnull(vptr) || !isnull(bptr) + with_gil(false) do + C.Py_DecRef(tptr) + C.Py_DecRef(vptr) + C.Py_DecRef(bptr) + end + end + end + end + return o + end +end +export PyException + +pythrow() = throw(PyException(Val(:new), C.PyErr_FetchTuple()...)) + +function Base.showerror(io::IO, e::PyException) + if isnull(e.tptr) + print(io, "Python: mysterious error (no error was actually set)") + return + end + + if CONFIG.sysautolasttraceback + err = C.Py_DecRef(C.PyImport_ImportModule("sys"), PYERR()) do sys + err = C.PyObject_SetAttrString(sys, "last_type", isnull(e.tptr) ? C.Py_None() : e.tptr) + ism1(err) && return PYERR() + err = C.PyObject_SetAttrString(sys, "last_value", isnull(e.vptr) ? C.Py_None() : e.vptr) + ism1(err) && return PYERR() + err = C.PyObject_SetAttrString(sys, "last_traceback", isnull(e.bptr) ? C.Py_None() : e.bptr) + ism1(err) && return PYERR() + nothing + end + if err == PYERR() + C.PyErr_Clear() + print(io, "") + end + end + + # # if this is a Julia exception then recursively print it and its stacktrace + # if pyerrmatches(e.t, pyjlexception) + # try + # jp = pyjlgetvalue(e.v.args[0]) + # if jp !== nothing + # je, jb = jp + # print(io, "Python: Julia: ") + # showerror(io, je) + # if jb === nothing + # println(io) + # print(io, "Stacktrace: none") + # else + # io2 = IOBuffer() + # Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) + # printstyled(io, String(take!(io2))) + # end + # end + # if pyisnone(e.b) + # println(io) + # printstyled(io, "Python stacktrace: none") + # return + # else + # @goto pystacktrace + # end + # catch err + # println(io, "") + # end + # end + + # otherwise, print the Python exception ****** TODO ****** + print(io, "Python: ") + + # print the type name + tname = C.Py_DecRef(C.PyObject_GetAttrString(e.tptr, "__name__")) do tnameo + C.PyUnicode_As(tnameo, String) + end + if tname === PYERR() + C.PyErr_Clear() + print(io, "") + else + print(io, tname) + end + + # print the error message + if !isnull(e.vptr) + print(io, ": ") + vstr = C.PyObject_StrAs(e.vptr, String) + if vstr === PYERR() + C.PyErr_Clear() + print(io, "") + else + print(io, vstr) + end + end + + # print the stacktrace + if !isnull(e.bptr) + @label pystacktrace + println(io) + printstyled(io, "Python stacktrace:") + err = C.Py_DecRef(C.PyImport_ImportModule("traceback")) do tb + C.Py_DecRef(C.PyObject_GetAttrString(tb, "extract_tb")) do extr + C.Py_DecRef(C.PyObject_CallNice(extr, C.PyObjectRef(e.bptr))) do fs + nfs = C.PySequence_Length(fs) + ism1(nfs) && return PYERR() + for i in 1:nfs + println(io) + printstyled(io, " [", i, "] ") + # name + err = C.Py_DecRef(C.PySequence_GetItem(fs, i-1)) do f + name = C.Py_DecRef(C.PyObject_GetAttrString(f, "name")) do nameo + C.PyObject_StrAs(nameo, String) + end + name === PYERR() && return PYERR() + printstyled(io, name, bold=true) + printstyled(io, " at ") + fname = C.Py_DecRef(C.PyObject_GetAttrString(f, "filename")) do fnameo + C.PyObject_StrAs(fnameo, String) + end + fname === PYERR() && return PYERR() + printstyled(io, fname, ":", bold=true) + lineno = C.Py_DecRef(C.PyObject_GetAttrString(f, "lineno")) do linenoo + C.PyObject_StrAs(linenoo, String) + end + lineno === PYERR() && return PYERR() + printstyled(io, lineno, bold=true) + nothing + end + err === PYERR() && return PYERR() + end + end + end + end + if err === PYERR() + C.PyErr_Clear() + print(io, "") + end + end +end diff --git a/src/Python.jl b/src/Python.jl index c0c22e8e..235d1132 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -52,84 +52,96 @@ Base.show(io::IO, ::MIME"text/plain", c::Config) = end const CONFIG = Config() +# Used to signal a Python error from functions that return general Julia objects +struct PYERR end + +# Used to signal that conversion failed +struct PYUNCONVERTED end + # C API -include("cpython.jl") +include("cpython/CPython.jl") const C = CPython const CPyPtr = C.PyPtr -struct CPyObjRef - ptr :: CPyPtr -end - -# core -include("object.jl") -include("error.jl") -include("import.jl") +# struct CPyObjRef +# ptr :: CPyPtr +# end + +# # core +# include("object.jl") +# include("error.jl") +# include("import.jl") include("gil.jl") -# abstract interfaces -include("number.jl") -include("sequence.jl") - -# fundamental objects -include("type.jl") -include("none.jl") - -# numeric objects -include("bool.jl") -include("int.jl") -include("float.jl") -include("complex.jl") - -# sequence objects -include("str.jl") -include("bytes.jl") -include("bytearray.jl") -include("tuple.jl") -include("list.jl") - -# mapping objects -include("dict.jl") -include("set.jl") - -# function objects -include("function.jl") - -# other objects -include("slice.jl") -include("range.jl") +include("PyException.jl") +include("PyCode.jl") +include("PyDict.jl") -# standard library -include("builtins.jl") include("eval.jl") -include("stdlib.jl") -include("io.jl") -include("fraction.jl") -include("datetime.jl") -include("collections.jl") - -# other packages -include("pandas.jl") -include("numpy.jl") -include("matplotlib.jl") - -# other Julia wrappers around Python values -include("PyIterable.jl") -include("PyList.jl") -include("PyDict.jl") -include("PySet.jl") -include("PyObjectArray.jl") -include("PyBuffer.jl") -include("PyArray.jl") -include("PyIO.jl") - -# other functionality -include("convert.jl") -include("newtype.jl") -include("julia.jl") -include("base.jl") -include("pywith.jl") -include("gui.jl") + +# # abstract interfaces +# include("number.jl") +# include("sequence.jl") + +# # fundamental objects +# include("type.jl") +# include("none.jl") + +# # numeric objects +# include("bool.jl") +# include("int.jl") +# include("float.jl") +# include("complex.jl") + +# # sequence objects +# include("str.jl") +# include("bytes.jl") +# include("bytearray.jl") +# include("tuple.jl") +# include("list.jl") + +# # mapping objects +# include("dict.jl") +# include("set.jl") + +# # function objects +# include("function.jl") + +# # other objects +# include("slice.jl") +# include("range.jl") + +# # standard library +# include("builtins.jl") +# include("eval.jl") +# include("stdlib.jl") +# include("io.jl") +# include("fraction.jl") +# include("datetime.jl") +# include("collections.jl") + +# # other packages +# include("pandas.jl") +# include("numpy.jl") +# include("matplotlib.jl") + +# # other Julia wrappers around Python values +# include("PyIterable.jl") +# include("PyList.jl") +# include("PyDict.jl") +# include("PySet.jl") +# include("PyObjectArray.jl") +# include("PyBuffer.jl") +# include("PyArray.jl") +# include("PyIO.jl") + +# # other functionality +# include("convert.jl") +# include("newtype.jl") +# include("julia.jl") +# include("base.jl") +# include("pywith.jl") +# include("gui.jl") # initialize include("init.jl") diff --git a/src/cpython.jl b/src/cpython.jl deleted file mode 100644 index c5f0582a..00000000 --- a/src/cpython.jl +++ /dev/null @@ -1,511 +0,0 @@ -module CPython - - using Libdl - using ..Python: CONFIG - using Base: @kwdef - using UnsafePointers: UnsafePtr - - @enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 - - const Py_LT = Cint(0) - const Py_LE = Cint(1) - const Py_EQ = Cint(2) - const Py_NE = Cint(3) - const Py_GT = Cint(4) - const Py_GE = Cint(5) - - const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments - const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs - const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) - const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) - const Py_METH_CLASS = 0x0010 # for class methods - const Py_METH_STATIC = 0x0020 # for static methods - - const Py_T_SHORT =0 - const Py_T_INT =1 - const Py_T_LONG =2 - const Py_T_FLOAT =3 - const Py_T_DOUBLE =4 - const Py_T_STRING =5 - const Py_T_OBJECT =6 - const Py_T_CHAR =7 - const Py_T_BYTE =8 - const Py_T_UBYTE =9 - const Py_T_USHORT =10 - const Py_T_UINT =11 - const Py_T_ULONG =12 - const Py_T_STRING_INPLACE =13 - const Py_T_BOOL =14 - const Py_T_OBJECT_EX =16 - const Py_T_LONGLONG =17 # added in Python 2.5 - const Py_T_ULONGLONG =18 # added in Python 2.5 - const Py_T_PYSSIZET =19 # added in Python 2.6 - const Py_T_NONE =20 # added in Python 3.0 - - const Py_READONLY = 1 - const Py_READ_RESTRICTED = 2 - const Py_WRITE_RESTRICTED = 4 - const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) - - const PyBUF_MAX_NDIM = 64 - - # Flags for getting buffers - const PyBUF_SIMPLE = 0x0 - const PyBUF_WRITABLE = 0x0001 - const PyBUF_WRITEABLE = PyBUF_WRITABLE - const PyBUF_FORMAT = 0x0004 - const PyBUF_ND = 0x0008 - const PyBUF_STRIDES = (0x0010 | PyBUF_ND) - const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) - const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) - const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) - const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) - - const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) - const PyBUF_CONTIG_RO = (PyBUF_ND) - - const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) - const PyBUF_STRIDED_RO = (PyBUF_STRIDES) - - const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) - const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) - - const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) - const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) - - const PyBUF_READ = 0x100 - const PyBUF_WRITE = 0x200 - - # Python 2.7 - const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) - const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) - const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 - const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) - const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) - const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) - const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) - const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) - const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) - const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) - const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) - const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) - - # Python 3.0+ has only these: - const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) - const Py_TPFLAGS_BASETYPE = (0x00000001<<10) - const Py_TPFLAGS_READY = (0x00000001<<12) - const Py_TPFLAGS_READYING = (0x00000001<<13) - const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) - const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) - const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) - const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) - const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) - const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) - const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) - const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) - const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) - const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) - const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) - const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) - const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) - - # only use this if we have the stackless extension - const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) - - const Py_hash_t = Cssize_t - const Py_ssize_t = Cssize_t - - @kwdef struct Py_complex - real :: Cdouble = 0.0 - imag :: Cdouble = 0.0 - end - - @kwdef struct PyObject - # assumes _PyObject_HEAD_EXTRA is empty - refcnt :: Py_ssize_t = 0 - type :: Ptr{PyObject} = C_NULL - end - - const PyPtr = Ptr{PyObject} - - @kwdef struct PyVarObject - ob_base :: PyObject = PyObject() - size :: Py_ssize_t = 0 - end - - @kwdef struct PyMethodDef - name :: Cstring = C_NULL - meth :: Ptr{Cvoid} = C_NULL - flags :: Cint = 0 - doc :: Cstring = C_NULL - end - - @kwdef struct PyGetSetDef - name :: Cstring = C_NULL - get :: Ptr{Cvoid} = C_NULL - set :: Ptr{Cvoid} = C_NULL - doc :: Cstring = C_NULL - closure :: Ptr{Cvoid} = C_NULL - end - - @kwdef struct PyMemberDef - name :: Cstring = C_NULL - typ :: Cint = 0 - offset :: Py_ssize_t = 0 - flags :: Cint = 0 - doc :: Cstring = C_NULL - end - - @kwdef struct PyNumberMethods - add :: Ptr{Cvoid} = C_NULL # (o,o)->o - subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o - power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - negative :: Ptr{Cvoid} = C_NULL # (o)->o - positive :: Ptr{Cvoid} = C_NULL # (o)->o - absolute :: Ptr{Cvoid} = C_NULL # (o)->o - bool :: Ptr{Cvoid} = C_NULL # (o)->Cint - invert :: Ptr{Cvoid} = C_NULL # (o)->o - lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - and :: Ptr{Cvoid} = C_NULL # (o,o)->o - xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - or :: Ptr{Cvoid} = C_NULL # (o,o)->o - int :: Ptr{Cvoid} = C_NULL # (o)->o - _reserved :: Ptr{Cvoid} = C_NULL - float :: Ptr{Cvoid} = C_NULL # (o)->o - inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o - floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - index :: Ptr{Cvoid} = C_NULL # (o)->o - matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - end - - @kwdef struct PySequenceMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - _was_item :: Ptr{Cvoid} = C_NULL - ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint - _was_ass_slice :: Ptr{Cvoid} = C_NULL - contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint - inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - end - - @kwdef struct PyMappingMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o - ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint - end - - @kwdef struct PyBufferProcs - get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint - release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid - end - - @kwdef struct Py_buffer - buf :: Ptr{Cvoid} = C_NULL - obj :: Ptr{Cvoid} = C_NULL - len :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - readonly :: Cint = 0 - ndim :: Cint = 0 - format :: Cstring = C_NULL - shape :: Ptr{Py_ssize_t} = C_NULL - strides :: Ptr{Py_ssize_t} = C_NULL - suboffsets :: Ptr{Py_ssize_t} = C_NULL - internal :: Ptr{Cvoid} = C_NULL - end - - @kwdef struct PyTypeObject - ob_base :: PyVarObject = PyVarObject() - name :: Cstring = C_NULL - - basicsize :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - - dealloc :: Ptr{Cvoid} = C_NULL - vectorcall_offset :: Py_ssize_t = 0 - getattr :: Ptr{Cvoid} = C_NULL - setattr :: Ptr{Cvoid} = C_NULL - as_async :: Ptr{Cvoid} = C_NULL - repr :: Ptr{Cvoid} = C_NULL - - as_number :: Ptr{PyNumberMethods} = C_NULL - as_sequence :: Ptr{PySequenceMethods} = C_NULL - as_mapping :: Ptr{PyMappingMethods} = C_NULL - - hash :: Ptr{Cvoid} = C_NULL - call :: Ptr{Cvoid} = C_NULL - str :: Ptr{Cvoid} = C_NULL - getattro :: Ptr{Cvoid} = C_NULL - setattro :: Ptr{Cvoid} = C_NULL - - as_buffer :: Ptr{PyBufferProcs} = C_NULL - - flags :: Culong = 0 - - doc :: Cstring = C_NULL - - traverse :: Ptr{Cvoid} = C_NULL - - clear :: Ptr{Cvoid} = C_NULL - - richcompare :: Ptr{Cvoid} = C_NULL - - weaklistoffset :: Py_ssize_t = 0 - - iter :: Ptr{Cvoid} = C_NULL - iternext :: Ptr{Cvoid} = C_NULL - - methods :: Ptr{PyMethodDef} = C_NULL - members :: Ptr{PyMemberDef} = C_NULL - getset :: Ptr{PyGetSetDef} = C_NULL - base :: PyPtr = C_NULL - dict :: PyPtr = C_NULL - descr_get :: Ptr{Cvoid} = C_NULL - descr_set :: Ptr{Cvoid} = C_NULL - dictoffset :: Py_ssize_t = 0 - init :: Ptr{Cvoid} = C_NULL - alloc :: Ptr{Cvoid} = C_NULL - new :: Ptr{Cvoid} = C_NULL - free :: Ptr{Cvoid} = C_NULL - is_gc :: Ptr{Cvoid} = C_NULL - bases :: PyPtr = C_NULL - mro :: PyPtr = C_NULL - cache :: PyPtr = C_NULL - subclasses :: PyPtr = C_NULL - weaklist :: PyPtr = C_NULL - del :: Ptr{Cvoid} = C_NULL - - version_tag :: Cuint = 0 - - finalize :: Ptr{Cvoid} = C_NULL - vectorcall :: Ptr{Cvoid} = C_NULL - - end - - const PyTypePtr = Ptr{PyTypeObject} - - pyglobal(name) = dlsym(CONFIG.libptr, name) - - macro cdef(name, rettype, argtypes) - name isa QuoteNode && name.value isa Symbol || error("name must be a symbol, got $name") - jname = esc(name.value) - refname = esc(Symbol(name.value, :_funcptr)) - name = esc(name) - rettype = esc(rettype) - argtypes isa Expr && argtypes.head==:tuple || error("argtypes must be a tuple, got $argtypes") - nargs = length(argtypes.args) - argtypes = esc(argtypes) - args = [gensym() for i in 1:nargs] - quote - const $refname = Ref(C_NULL) - function $jname($(args...)) - ptr = $refname[] - if ptr == C_NULL - ptr = $refname[] = pyglobal($name) - end - ccall(ptr, $rettype, $argtypes, $(args...)) - end - end - end - - @cdef :Py_Initialize Cvoid () - @cdef :Py_InitializeEx Cvoid (Cint,) - @cdef :Py_Finalize Cvoid () - @cdef :Py_FinalizeEx Cint () - @cdef :Py_AtExit Cint (Ptr{Cvoid},) - @cdef :Py_IsInitialized Cint () - - @cdef :Py_SetPythonHome Cvoid (Cwstring,) - @cdef :Py_SetProgramName Cvoid (Cwstring,) - @cdef :Py_GetVersion Cstring () - - @cdef :Py_IncRef Cvoid (PyPtr,) - @cdef :Py_DecRef Cvoid (PyPtr,) - - @cdef :PyEval_SaveThread Ptr{Cvoid} () - @cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) - - @cdef :PyGILState_Ensure PyGILState_STATE () - @cdef :PyGILState_Release Cvoid (PyGILState_STATE,) - - @cdef :PyImport_ImportModule PyPtr (Cstring,) - @cdef :PyImport_Import PyPtr (PyPtr,) - - @cdef :PyErr_Occurred PyPtr () - @cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) - @cdef :PyErr_Clear Cvoid () - @cdef :PyErr_SetNone Cvoid (PyPtr,) - @cdef :PyErr_SetString Cvoid (PyPtr, Cstring) - @cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) - @cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) - @cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) - @cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) - - @cdef :_PyObject_New PyPtr (PyPtr,) - @cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) - @cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) - @cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) - @cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) - @cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) - @cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) - @cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) - @cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) - @cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) - @cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) - @cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) - @cdef :PyObject_Repr PyPtr (PyPtr,) - @cdef :PyObject_ASCII PyPtr (PyPtr,) - @cdef :PyObject_Str PyPtr (PyPtr,) - @cdef :PyObject_Bytes PyPtr (PyPtr,) - @cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) - @cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) - @cdef :PyObject_Hash Py_hash_t (PyPtr,) - @cdef :PyObject_IsTrue Cint (PyPtr,) - @cdef :PyObject_Length Py_ssize_t (PyPtr,) - @cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) - @cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_DelItem Cint (PyPtr, PyPtr) - @cdef :PyObject_Dir PyPtr (PyPtr,) - @cdef :PyObject_GetIter PyPtr (PyPtr,) - @cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) - - @cdef :PySequence_Contains Cint (PyPtr, PyPtr) - - @cdef :PyInstanceMethod_New PyPtr (PyPtr,) - - @cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) - @cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) - - @cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) - @cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) - - @cdef :PyTuple_New PyPtr (Py_ssize_t,) - @cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) - - @cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) - @cdef :PyType_Ready Cint (PyPtr,) - - @cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyNumber_Negative PyPtr (PyPtr,) - @cdef :PyNumber_Positive PyPtr (PyPtr,) - @cdef :PyNumber_Absolute PyPtr (PyPtr,) - @cdef :PyNumber_Invert PyPtr (PyPtr,) - @cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_And PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Long PyPtr (PyPtr,) - @cdef :PyNumber_Float PyPtr (PyPtr,) - @cdef :PyNumber_Index PyPtr (PyPtr,) - - @cdef :PyIter_Next PyPtr (PyPtr,) - - @cdef :PyLong_FromLongLong PyPtr (Clonglong,) - @cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) - @cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) - @cdef :PyLong_AsLongLong Clonglong (PyPtr,) - @cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) - - @cdef :PyFloat_FromDouble PyPtr (Cdouble,) - @cdef :PyFloat_AsDouble Cdouble (PyPtr,) - - @cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) - @cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) - - @cdef :PyList_New PyPtr (Py_ssize_t,) - @cdef :PyList_Append Cint (PyPtr, PyPtr) - @cdef :PyList_AsTuple PyPtr (PyPtr,) - - @cdef :PyDict_New PyPtr () - @cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) - @cdef :PyDict_DelItemString Cint (PyPtr, Cstring) - - Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] - Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] - Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) - Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) - Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) - - PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] - PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) - PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) - - function PyObject_CheckBuffer(o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - p != C_NULL && p.get[] != C_NULL - end - - function PyObject_GetBuffer(o, b, flags) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if p == C_NULL || p.get[] == C_NULL - PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") - return Cint(-1) - end - ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) - end - - function PyBuffer_Release(_b) - b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) - o = b.obj[] - o == C_NULL && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (p != C_NULL && p.release[] != C_NULL) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) - end - b.obj[] = C_NULL - Py_DecRef(o) - return - end - - function PyOS_RunInputHook() - hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) - hook == C_NULL || ccall(hook, Cint, ()) - nothing - end - -end diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl new file mode 100644 index 00000000..bb61e102 --- /dev/null +++ b/src/cpython/CPython.jl @@ -0,0 +1,833 @@ +module CPython + +using Libdl +using ..Python: CONFIG, isnull, ism1, PYERR, PYUNCONVERTED +using Base: @kwdef +using UnsafePointers: UnsafePtr + +@enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 + +const Py_single_input = 256 +const Py_file_input = 257 +const Py_eval_input = 258 +const Py_func_type_input = 345 + +const Py_LT = Cint(0) +const Py_LE = Cint(1) +const Py_EQ = Cint(2) +const Py_NE = Cint(3) +const Py_GT = Cint(4) +const Py_GE = Cint(5) + +const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments +const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs +const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) +const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) +const Py_METH_CLASS = 0x0010 # for class methods +const Py_METH_STATIC = 0x0020 # for static methods + +const Py_T_SHORT =0 +const Py_T_INT =1 +const Py_T_LONG =2 +const Py_T_FLOAT =3 +const Py_T_DOUBLE =4 +const Py_T_STRING =5 +const Py_T_OBJECT =6 +const Py_T_CHAR =7 +const Py_T_BYTE =8 +const Py_T_UBYTE =9 +const Py_T_USHORT =10 +const Py_T_UINT =11 +const Py_T_ULONG =12 +const Py_T_STRING_INPLACE =13 +const Py_T_BOOL =14 +const Py_T_OBJECT_EX =16 +const Py_T_LONGLONG =17 # added in Python 2.5 +const Py_T_ULONGLONG =18 # added in Python 2.5 +const Py_T_PYSSIZET =19 # added in Python 2.6 +const Py_T_NONE =20 # added in Python 3.0 + +const Py_READONLY = 1 +const Py_READ_RESTRICTED = 2 +const Py_WRITE_RESTRICTED = 4 +const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) + +const PyBUF_MAX_NDIM = 64 + +# Flags for getting buffers +const PyBUF_SIMPLE = 0x0 +const PyBUF_WRITABLE = 0x0001 +const PyBUF_WRITEABLE = PyBUF_WRITABLE +const PyBUF_FORMAT = 0x0004 +const PyBUF_ND = 0x0008 +const PyBUF_STRIDES = (0x0010 | PyBUF_ND) +const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) +const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) +const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) +const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) + +const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) +const PyBUF_CONTIG_RO = (PyBUF_ND) + +const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) +const PyBUF_STRIDED_RO = (PyBUF_STRIDES) + +const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) + +const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) + +const PyBUF_READ = 0x100 +const PyBUF_WRITE = 0x200 + +# Python 2.7 +const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) +const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) +const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 +const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) +const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) +const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) +const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) +const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) +const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) +const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) +const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) +const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) + +# Python 3.0+ has only these: +const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) +const Py_TPFLAGS_BASETYPE = (0x00000001<<10) +const Py_TPFLAGS_READY = (0x00000001<<12) +const Py_TPFLAGS_READYING = (0x00000001<<13) +const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) +const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) +const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) +const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) +const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) +const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) +const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) +const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) +const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) +const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) +const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) +const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) +const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) + +# only use this if we have the stackless extension +const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) + +const Py_hash_t = Cssize_t +const Py_ssize_t = Cssize_t + +@kwdef struct Py_complex + real :: Cdouble = 0.0 + imag :: Cdouble = 0.0 +end + +@kwdef struct PyObject + # assumes _PyObject_HEAD_EXTRA is empty + refcnt :: Py_ssize_t = 0 + type :: Ptr{PyObject} = C_NULL +end + +const PyPtr = Ptr{PyObject} + +struct PyObjectRef + ptr :: PyPtr +end +Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr + +@kwdef struct PyVarObject + ob_base :: PyObject = PyObject() + size :: Py_ssize_t = 0 +end + +@kwdef struct PyMethodDef + name :: Cstring = C_NULL + meth :: Ptr{Cvoid} = C_NULL + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyGetSetDef + name :: Cstring = C_NULL + get :: Ptr{Cvoid} = C_NULL + set :: Ptr{Cvoid} = C_NULL + doc :: Cstring = C_NULL + closure :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyMemberDef + name :: Cstring = C_NULL + typ :: Cint = 0 + offset :: Py_ssize_t = 0 + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyNumberMethods + add :: Ptr{Cvoid} = C_NULL # (o,o)->o + subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o + power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + negative :: Ptr{Cvoid} = C_NULL # (o)->o + positive :: Ptr{Cvoid} = C_NULL # (o)->o + absolute :: Ptr{Cvoid} = C_NULL # (o)->o + bool :: Ptr{Cvoid} = C_NULL # (o)->Cint + invert :: Ptr{Cvoid} = C_NULL # (o)->o + lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + and :: Ptr{Cvoid} = C_NULL # (o,o)->o + xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + or :: Ptr{Cvoid} = C_NULL # (o,o)->o + int :: Ptr{Cvoid} = C_NULL # (o)->o + _reserved :: Ptr{Cvoid} = C_NULL + float :: Ptr{Cvoid} = C_NULL # (o)->o + inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o + floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + index :: Ptr{Cvoid} = C_NULL # (o)->o + matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o +end + +@kwdef struct PySequenceMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + _was_item :: Ptr{Cvoid} = C_NULL + ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint + _was_ass_slice :: Ptr{Cvoid} = C_NULL + contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint + inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o +end + +@kwdef struct PyMappingMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o + ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint +end + +@kwdef struct PyBufferProcs + get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint + release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid +end + +@kwdef struct Py_buffer + buf :: Ptr{Cvoid} = C_NULL + obj :: Ptr{Cvoid} = C_NULL + len :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + readonly :: Cint = 0 + ndim :: Cint = 0 + format :: Cstring = C_NULL + shape :: Ptr{Py_ssize_t} = C_NULL + strides :: Ptr{Py_ssize_t} = C_NULL + suboffsets :: Ptr{Py_ssize_t} = C_NULL + internal :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyTypeObject + ob_base :: PyVarObject = PyVarObject() + name :: Cstring = C_NULL + + basicsize :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + + dealloc :: Ptr{Cvoid} = C_NULL + vectorcall_offset :: Py_ssize_t = 0 + getattr :: Ptr{Cvoid} = C_NULL + setattr :: Ptr{Cvoid} = C_NULL + as_async :: Ptr{Cvoid} = C_NULL + repr :: Ptr{Cvoid} = C_NULL + + as_number :: Ptr{PyNumberMethods} = C_NULL + as_sequence :: Ptr{PySequenceMethods} = C_NULL + as_mapping :: Ptr{PyMappingMethods} = C_NULL + + hash :: Ptr{Cvoid} = C_NULL + call :: Ptr{Cvoid} = C_NULL + str :: Ptr{Cvoid} = C_NULL + getattro :: Ptr{Cvoid} = C_NULL + setattro :: Ptr{Cvoid} = C_NULL + + as_buffer :: Ptr{PyBufferProcs} = C_NULL + + flags :: Culong = 0 + + doc :: Cstring = C_NULL + + traverse :: Ptr{Cvoid} = C_NULL + + clear :: Ptr{Cvoid} = C_NULL + + richcompare :: Ptr{Cvoid} = C_NULL + + weaklistoffset :: Py_ssize_t = 0 + + iter :: Ptr{Cvoid} = C_NULL + iternext :: Ptr{Cvoid} = C_NULL + + methods :: Ptr{PyMethodDef} = C_NULL + members :: Ptr{PyMemberDef} = C_NULL + getset :: Ptr{PyGetSetDef} = C_NULL + base :: PyPtr = C_NULL + dict :: PyPtr = C_NULL + descr_get :: Ptr{Cvoid} = C_NULL + descr_set :: Ptr{Cvoid} = C_NULL + dictoffset :: Py_ssize_t = 0 + init :: Ptr{Cvoid} = C_NULL + alloc :: Ptr{Cvoid} = C_NULL + new :: Ptr{Cvoid} = C_NULL + free :: Ptr{Cvoid} = C_NULL + is_gc :: Ptr{Cvoid} = C_NULL + bases :: PyPtr = C_NULL + mro :: PyPtr = C_NULL + cache :: PyPtr = C_NULL + subclasses :: PyPtr = C_NULL + weaklist :: PyPtr = C_NULL + del :: Ptr{Cvoid} = C_NULL + + version_tag :: Cuint = 0 + + finalize :: Ptr{Cvoid} = C_NULL + vectorcall :: Ptr{Cvoid} = C_NULL + +end + +const PyTypePtr = Ptr{PyTypeObject} + +pyglobal(name) = dlsym(CONFIG.libptr, name) +pyglobal(r::Ref{Ptr{Cvoid}}, name) = (p=r[]; if isnull(p); p=r[]=pyglobal(name); end; p) + +pyloadglobal(r::Ref{Ptr{T}}, name) where {T} = (p=r[]; if isnull(p); p=r[]=unsafe_load(Ptr{Ptr{T}}(pyglobal(name))) end; p) + +macro cdef(name, rettype, argtypes) + name isa QuoteNode && name.value isa Symbol || error("name must be a symbol, got $name") + jname = esc(name.value) + refname = esc(Symbol(name.value, :__ref)) + name = esc(name) + rettype = esc(rettype) + argtypes isa Expr && argtypes.head==:tuple || error("argtypes must be a tuple, got $argtypes") + nargs = length(argtypes.args) + argtypes = esc(argtypes) + args = [gensym() for i in 1:nargs] + quote + const $refname = Ref(C_NULL) + $jname($(args...)) = ccall(pyglobal($refname, $name), $rettype, $argtypes, $(args...)) + end +end + +### INITIALIZE + +@cdef :Py_Initialize Cvoid () +@cdef :Py_InitializeEx Cvoid (Cint,) +@cdef :Py_Finalize Cvoid () +@cdef :Py_FinalizeEx Cint () +@cdef :Py_AtExit Cint (Ptr{Cvoid},) +@cdef :Py_IsInitialized Cint () + +@cdef :Py_SetPythonHome Cvoid (Cwstring,) +@cdef :Py_SetProgramName Cvoid (Cwstring,) +@cdef :Py_GetVersion Cstring () + +### REFCOUNT + +@cdef :Py_IncRef Cvoid (PyPtr,) +@cdef :Py_DecRef Cvoid (PyPtr,) +Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] + +Py_DecRef(f::Function, o::Ptr, dflt=PYERR()) = + isnull(o) ? dflt : (r=f(o); Py_DecRef(o); r) + +Py_Is(o1, o2) = Base.unsafe_convert(PyPtr, o1) == Base.unsafe_convert(PyPtr, o2) + +### EVAL + +@cdef :PyEval_EvalCode PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :Py_CompileString PyPtr (Cstring, Cstring, Cint) +@cdef :PyEval_GetBuiltins PyPtr () + +### GIL & THREADS + +@cdef :PyEval_SaveThread Ptr{Cvoid} () +@cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) +@cdef :PyGILState_Ensure PyGILState_STATE () +@cdef :PyGILState_Release Cvoid (PyGILState_STATE,) + +### IMPORT + +@cdef :PyImport_ImportModule PyPtr (Cstring,) +@cdef :PyImport_Import PyPtr (PyPtr,) + +### ERRORS + +@cdef :PyErr_Occurred PyPtr () +@cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) +@cdef :PyErr_Clear Cvoid () +@cdef :PyErr_SetNone Cvoid (PyPtr,) +@cdef :PyErr_SetString Cvoid (PyPtr, Cstring) +@cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) +@cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) + +PyErr_IsSet() = !isnull(PyErr_Occurred()) + +PyErr_IsSet(t) = (o=PyErr_Occurred(); !isnull(o) && PyErr_GivenExceptionMatches(o,t)!=0) + +function PyErr_FetchTuple(normalize::Bool=false) + t = Ref{PyPtr}() + v = Ref{PyPtr}() + b = Ref{PyPtr}() + PyErr_Fetch(t, v, b) + normalize && PyErr_NormalizeException(t, v, b) + (t[], v[], b[]) +end + +### EXCEPTIONS + +const PyExc_TypeError__ref = Ref(PyPtr()) +PyExc_TypeError() = pyloadglobal(PyExc_TypeError__ref, :PyExc_TypeError) + +### NONE + +const Py_None__ref = Ref(C_NULL) +Py_None() = PyPtr(pyglobal(Py_None__ref, :_Py_NoneStruct)) + +PyNone_Check(o) = Py_Is(o, Py_None()) + +PyNone_New() = (o=Py_None(); Py_IncRef(o); o) + +PyNone_As(o, ::Type{T}) where {T} = + if Nothing <: T + nothing + elseif Missing <: T + missing + else + PYUNCONVERTED() + end + +### OBJECT + +@cdef :_PyObject_New PyPtr (PyPtr,) +@cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) +@cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) +@cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) +@cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) +@cdef :PyObject_Repr PyPtr (PyPtr,) +@cdef :PyObject_ASCII PyPtr (PyPtr,) +@cdef :PyObject_Str PyPtr (PyPtr,) +@cdef :PyObject_Bytes PyPtr (PyPtr,) +@cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) +@cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) +@cdef :PyObject_Hash Py_hash_t (PyPtr,) +@cdef :PyObject_IsTrue Cint (PyPtr,) +@cdef :PyObject_Length Py_ssize_t (PyPtr,) +@cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelItem Cint (PyPtr, PyPtr) +@cdef :PyObject_Dir PyPtr (PyPtr,) +@cdef :PyObject_GetIter PyPtr (PyPtr,) +@cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) + +function PyObject_ReprAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} + x = PyObject_Repr(o) + isnull(x) && return PYERR() + r = PyUnicode_As(x, T) + Py_DecRef(x) + r +end + +function PyObject_StrAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} + x = PyObject_Str(o) + isnull(x) && return PYERR() + r = PyUnicode_As(x, T) + Py_DecRef(x) + r +end + +function PyObject_ASCIIAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} + x = PyObject_ASCII(o) + isnull(x) && return PYERR() + r = PyUnicode_As(x, T) + Py_DecRef(x) + r +end + +function PyObject_BytesAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} + x = PyObject_Bytes(o) + isnull(x) && return PYERR() + r = PyBytes_As(x, T) + Py_DecRef(x) + r +end + +PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) +PyObject_From(x::Nothing) = PyNone_New() +PyObject_From(x::Bool) = PyBool_From(x) +PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) +PyObject_From(x::Tuple) = PyTuple_From(x) +function PyObject_From(x) + PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") + PyPtr() +end + +function PyObject_CallArgs(f, args, kwargs=()) + if !isempty(kwargs) + error("kwargs not implemented") + elseif !isempty(args) + argso = PyTuple_From(Tuple(args)) + isnull(argso) && return PyPtr() + r = PyObject_CallObject(f, argso) + Py_DecRef(argso) + return r + else + PyObject_CallObject(f, C_NULL) + end +end + +PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) + +const PYOBJECT_AS_RULES = Dict{PyPtr, Union{Nothing,Function}}() +const PYOBJECT_AS_CRULES = Dict{Type, Dict{PyPtr, Ptr{Cvoid}}}() +const PYOBJECT_AS_CRULES_CACHE = Dict{Type, Dict{PyPtr, Any}}() + +function PyObject_As__rule(t::PyPtr) + name = Py_DecRef(PyObject_GetAttrString(t, "__name__")) do tnameo + Py_DecRef(PyObject_GetAttrString(t, "__module__")) do mnameo + tname = PyUnicode_As(tnameo, String) + tname === PYERR() && return PYERR() + mname = PyUnicode_As(mnameo, String) + mname === PYERR() && return PYERR() + "$mname.$tname" + end + end + name == PYERR() && return PYERR() + @show name + PyObject_As__rule(t, Val(Symbol(name))) +end + +PyObject_As__rule(::PyPtr, ::Val) = nothing +PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.NoneType")}) = PyNone_As +PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.bool")}) = PyBool_As +PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.str")}) = PyUnicode_As +PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.bytes")}) = PyBytes_As + +struct PyObject_As__crule_struct{T,F} + f :: F +end +function (r::PyObject_As__crule_struct{T,F})(o::PyPtr, ptr::Ptr) where {T,F} + res = r.f(o, T) + if res === PYERR() + return Cint(2) + elseif res === PYUNCONVERTED() + return Cint(1) + else + unsafe_store!(ptr, res) + return Cint(0) + end +end + +function PyObject_As(o::PyPtr, ::Type{T}) where {T} + # MRO + mro = PyType_MRO(Py_Type(o)) + ref = Ref{Any}() + rules = get!(Dict{PyPtr, Ptr{Cvoid}}, PYOBJECT_AS_CRULES, T)::Dict{PyPtr,Ptr{Cvoid}} + for i in 1:PyTuple_Size(mro) + t = PyTuple_GetItem(mro, i-1) + isnull(t) && return PYERR() + crule = get(rules, t, nothing) + if crule === nothing + rule = PyObject_As__rule(t) + rule === PYERR() && return PYERR() + if rule === nothing + rules[t] = C_NULL + continue + else + crulefunc = @cfunction($(PyObject_As__crule_struct{T, typeof(rule)}(rule)), Cint, (PyPtr, Ptr{Any})) + get!(Dict{PyPtr, Any}, PYOBJECT_AS_CRULES_CACHE, T)[t] = crulefunc + crule = rules[t] = Base.unsafe_convert(Ptr{Cvoid}, crulefunc) + end + elseif crule == C_NULL + continue + end + res = ccall(crule, Cint, (PyPtr, Ptr{Any}), o, ref) + if res == 0 + return ref[]::T + elseif res == 2 + return PYERR() + end + end + PYUNCONVERTED() +end +PyObject_As(o, ::Type{T}) where {T} = GC.@preserve o PyObject_As(Base.unsafe_convert(PyPtr, o), T) + +### SEQUENCE + +@cdef :PySequence_Length Py_ssize_t (PyPtr,) +@cdef :PySequence_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PySequence_Contains Cint (PyPtr, PyPtr) + +### MAPPING + +@cdef :PyMapping_HasKeyString Cint (PyPtr, Cstring) +@cdef :PyMapping_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyMapping_GetItemString PyPtr (PyPtr, Cstring) + +### METHOD + +@cdef :PyInstanceMethod_New PyPtr (PyPtr,) + +### STR + +@cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) +@cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) + +const PyUnicode_Type__ref = Ref{PyPtr}() +PyUnicode_Type() = pyloadglobal(PyUnicode_Type__ref, :PyUnicode_Type) + +PyUnicode_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_UNICODE_SUBCLASS) +PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) + +PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) + +PyUnicode_As(o, ::Type) = PYUNCONVERTED() + +function PyUnicode_As(o, ::Type{T}) where {T<:Union{String,Vector{Int8},Vector{UInt8}}} + b = PyUnicode_AsUTF8String(o) + isnull(b) && return PYERR() + r = PyBytes_As(b, T) + Py_DecRef(b) + r +end + +### BYTES + +@cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) +@cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) + +PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyBytes_FromStringAndSize(pointer(s), sizeof(s)) + +PyBytes_As(o, ::Type) = PYUNCONVERTED() + +function PyBytes_As(o, ::Type{String}) + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return PYERR() + Base.unsafe_string(ptr[], len[]) +end + +function PyBytes_As(o, ::Type{Vector{T}}) where {T<:Union{UInt8,Int8}} + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return PYERR() + copy(Base.unsafe_wrap(Vector{T}, Ptr{T}(ptr[]), len[])) +end + +### TUPLE + +@cdef :PyTuple_New PyPtr (Py_ssize_t,) +@cdef :PyTuple_Size Py_ssize_t (PyPtr,) +@cdef :PyTuple_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) + +function PyTuple_From(xs::Tuple) + t = PyTuple_New(length(xs)) + isnull(t) && return PyPtr() + for (i,x) in enumerate(xs) + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(t); return PyPtr()) + err = PyTuple_SetItem(t, i-1, xo) # steals xo + ism1(err) && (Py_DecRef(t); return PyPtr()) + end + return t +end + +### TYPE + +@cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) +@cdef :PyType_Ready Cint (PyPtr,) + +Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] +Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) +Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) +Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) + +PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] +PyType_Name(o) = GC.@preserve o unsafe_string(UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).name[!]) +PyType_MRO(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).mro[!] + +PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) +PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) + +### NUMBER + +@cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_Negative PyPtr (PyPtr,) +@cdef :PyNumber_Positive PyPtr (PyPtr,) +@cdef :PyNumber_Absolute PyPtr (PyPtr,) +@cdef :PyNumber_Invert PyPtr (PyPtr,) +@cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_And PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Long PyPtr (PyPtr,) +@cdef :PyNumber_Float PyPtr (PyPtr,) +@cdef :PyNumber_Index PyPtr (PyPtr,) + +### ITER + +@cdef :PyIter_Next PyPtr (PyPtr,) + +### BOOL + +const PyBool_Type__ref = Ref(C_NULL) +PyBool_Type() = PyPtr(pyglobal(PyBool_Type__ref, :PyBool_Type)) + +const Py_True__ref = Ref(C_NULL) +Py_True() = PyPtr(pyglobal(Py_True__ref, :_Py_TrueStruct)) + +const Py_False__ref = Ref(C_NULL) +Py_False() = PyPtr(pyglobal(Py_False__ref, :_Py_FalseStruct)) + +PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) + +PyBool_Check(o) = Py_Type(o) == PyBool_Type() + +PyBool_As(o, ::Type{T}) where {T} = + if Bool <: T + if Py_Is(o, Py_True()) + true + elseif Py_Is(o, Py_False()) + false + else + PyErr_SetString(PyExc_TypeError(), "not a 'bool'") + PYERR() + end + else + PYUNCONVERTED() + end + + +### INT + +@cdef :PyLong_FromLongLong PyPtr (Clonglong,) +@cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) +@cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) +@cdef :PyLong_AsLongLong Clonglong (PyPtr,) +@cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) + +### FLOAT + +@cdef :PyFloat_FromDouble PyPtr (Cdouble,) +@cdef :PyFloat_AsDouble Cdouble (PyPtr,) + +### COMPLEX + +@cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) +@cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) + +### LIST + +@cdef :PyList_New PyPtr (Py_ssize_t,) +@cdef :PyList_Append Cint (PyPtr, PyPtr) +@cdef :PyList_AsTuple PyPtr (PyPtr,) + +### DICT + +@cdef :PyDict_New PyPtr () +@cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyDict_DelItemString Cint (PyPtr, Cstring) + +### BUFFER + +function PyObject_CheckBuffer(o) + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + !isnull(p) && !isnull(p.get[]) +end + +function PyObject_GetBuffer(o, b, flags) + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if isnull(p) || isnull(p.get[]) + PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") + return Cint(-1) + end + ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) +end + +function PyBuffer_Release(_b) + b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) + o = b.obj[] + isnull(o) && return + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if (!isnull(p) && !isnull(p.release[])) + ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) + end + b.obj[] = C_NULL + Py_DecRef(o) + return +end + +### INPUT HOOK + +function PyOS_RunInputHook() + hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) + isnull(hook) || ccall(hook, Cint, ()) + nothing +end + +end diff --git a/src/eval.jl b/src/eval.jl index 0e28a9bc..8780f57b 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,137 +1,89 @@ -const SCOPES = Dict{Any,PyObject}() -const COMPILECACHE = Dict{Tuple{String,String,String},PyObject}() - -scope(s) = get!(pydict, SCOPES, s) -scope(s::PyObject) = s - -""" - pyeval(src, scope, locals=nothing) - -Evaluate Python expression `src` in the context of the given `scope` and `locals`. Return the value. - -If the `scope` is a Python object, it must be a `dict` to use as globals. - -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyeval(src, @__MODULE__)` evaluates in a scope unique to the current module. -""" -pyeval(src, globals, locals=nothing) = pyevalfunc(src, scope(globals), locals) - -""" - pyexec(src, scope, locals=nothing) - -Execute Python expression `src` in the context of the given `scope` and `locals`. - -If the `scope` is a Python object, it must be a `dict` to use as globals. - -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyexec(src, @__MODULE__)` executes in a scope unique to the current module. -""" -pyexec(src, globals, locals=nothing) = (pyexecfunc(src, scope(globals), locals); nothing) - -""" - pyevaldb(src, scope, locals=nothing) - -Same as `pyeval(src, scope, locals)` but evaluated inside a `pdb` debugger. -""" -pyevaldb(src, globals, locals=nothing) = pypdbmodule.runeval(src, scope(globals), locals) - -""" - pyexecdb(src, scope, locals=nothing) - -Same as `pyexec(src, scope, locals)` but evaluated inside a `pdb` debugger. -""" -pyexecdb(src, globals, locals=nothing) = (pypdbmodule.run(src, scope(globals), locals); nothing) - -""" - py"...."[flags] - -Evaluate (`v`) or execute (`x`) the given Python source code. - -Julia values may be interpolated into the source code with `\$` syntax. For a literal `\$`, enter `\$\$`. - -Execution occurs in a global scope unique to the current module. - -The flags can be any combination of the following characters: -- `v` (evaluate): Evaluate a single expression and return its value. -- `x` (execute): Execute the code and return `nothing`. -- `g` (globals): Return the dict of globals for the scope instead. -- `l` (locals): Perform the computation in a new local scope and return that scope. -- `c` (compile): Cache a compiled version of the source and re-use it each time, for speed. -- `d` (debug): Run inside a `pdb` debugger. - -If neither `v` nor `x` is specified and the code is a single line then `v` (evaluate) is assumed, otherwise `x` (execute). -""" -macro py_str(src::String, flags::String="") - # parse the flags - exec = '\n' in src - retglobals = false - retlocals = false - lazy = false - compile = false - debug = false - for f in flags - if f == 'x' - exec = true - elseif f == 'v' - exec = false - elseif f == 'g' - retglobals = true - elseif f == 'l' - retlocals = true - # elseif f == 'z' - # lazy = true - elseif f == 'c' - compile = true - elseif f == 'd' - debug = true - else - error("invalid flags: `py\"...\"$flags`") +function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Nothing}=globals, extras::Union{NamedTuple,Nothing}=nothing) where {R} + # get code + cptr = pyptr(co) + # get globals & ensure __builtins__ is set + gptr = pyptr(globals) + if !globals.hasbuiltins + if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 + err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) + ism1(err) && pythrow() end + globals.hasbuiltins = true end - # parse src for $-interpolations - chunks = String[] - interps = [] - i = firstindex(src) - while true - j = findnext('$', src, i) - if j === nothing - push!(chunks, src[i:end]) - break - else - push!(chunks, src[i:prevind(src,j)]) - end - if checkbounds(Bool, src, j+1) && src[j+1] == '$' - push!(chunks, "\$") - i = j+2 - else - ex, i = Meta.parse(src, j+1, greedy=false) - var = "_jl_interp_$(length(interps)+1)_" - push!(interps, var => ex) - push!(chunks, "($var)") + # get locals (ALLOCATES lptr if locals===nothing) + if locals === nothing + lptr = C.PyDict_New() + isnull(lptr) && pythrow() + else + lptr = pyptr(locals) + end + # insert extra locals + if extras !== nothing + for (k,v) in pairs(extras) + vo = C.PyObject_From(v) + if isnull(vo) + locals===nothing && C.Py_DecRef(lptr) + pythrow() + end + err = C.PyMapping_SetItemString(lptr, string(k), vo) + C.Py_DecRef(vo) + if ism1(err) + locals===nothing && C.Py_DecRef(lptr) + pythrow() + end end end - newsrcstr = newsrc = join(chunks) - # compile the code so there is string information - cfile = "julia:$(__source__.file):$(__source__.line)" - cmode = exec ? "exec" : "eval" - newsrc = :(pycompile($newsrcstr, $cfile, $cmode)) - if compile - # compile the code lazily (so the python parser is only invoked once) - # Julia crashes if you try to put pylazyobject(()->pycompile(...)) directly in the syntax tree. Is this a bug?? - # Instead, we use a run-time lookup table. - newsrc = :(get!(()->$newsrc, COMPILECACHE, ($newsrcstr, $cfile, $cmode))) + # Call eval (ALLOCATES rptr) + rptr = C.PyEval_EvalCode(cptr, gptr, lptr) + if isnull(rptr) + locals === nothing && C.Py_DecRef(lptr) + pythrow() end - # make the expression - ex = :(let - globals = scope(@__MODULE__) - locals = $(retlocals ? :(pydict()) : :globals) - $([:(check(C.PyDict_SetItemString(globals, $k, pyobject($(esc(v)))))) for (k,v) in interps]...) - result = $(debug ? (exec ? :pyexecdb : :pyevaldb) : (exec ? :pyexec : :pyeval))($newsrc, globals, locals) - $([:(check(C.PyDict_DelItemString(globals, $k))) for (k,v) in interps]...) - $(retlocals ? :locals : retglobals ? :globals : exec ? nothing : :result) - end) - if lazy - # wrap as a lazy object - ex = :(pylazyobject(() -> $ex)) + # TODO: convert rptr using PyObject_As + if co.mode == :exec + if R <: Nothing + C.Py_DecRef(rptr) + locals===nothing && C.Py_DecRef(lptr) + return nothing + elseif R <: NamedTuple && isconcretetype(R) + C.Py_DecRef(rptr) + locals===nothing && C.Py_DecRef(lptr) + error("returning NamedTuple not implemented yet") + else + C.Py_DecRef(rptr) + locals===nothing && C.Py_DecRef(lptr) + error("invalid return type $(R)") + end + elseif co.mode == :eval + ret = C.PyObject_As(rptr, R) + ret === PYUNCONVERTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert this '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$R'") + C.Py_DecRef(rptr) + locals===nothing && C.Py_DecRef(lptr) + ret === PYERR() && pythrow() + ret === PYUNCONVERTED() && pythrow() + return ret + else + C.Py_DecRef(rptr) + locals===nothing && C.Py_DecRef(lptr) + error("invalid mode $(repr(co.mode))") end - ex end -export @py_str +export pyeval + +module CompiledCode end + +macro pyeval(R, code::String, locals=nothing) + co = PyCode(code, "", :eval) + nm = gensym() + Base.eval(CompiledCode, :($nm = $co)) + :(pyeval($(esc(R)), $co, $(esc(:pyglobals)), $(esc(locals)))) +end +export @pyeval + +macro pyexec(code::String, locals=nothing) + co = PyCode(code, "", :exec) + nm = gensym() + Base.eval(CompiledCode, :($nm = $co)) + :(pyeval(Nothing, $co, $(esc(:pyglobals)), $(esc(locals)))) +end +export @pyexec diff --git a/src/init.jl b/src/init.jl index f7eb725f..2b728129 100644 --- a/src/init.jl +++ b/src/init.jl @@ -1,9 +1,3 @@ -function python_cmd(args) - env = copy(ENV) - env["PYTHONIOENCODING"] = "UTF-8" - setenv(`$(CONFIG.exepath) $args`, env) -end - function __init__() # Check if libpython is already loaded (i.e. if the Julia interpreter was started from a Python process) CONFIG.isembedded = haskey(ENV, "PYTHONJL_LIBPTR") @@ -43,6 +37,13 @@ function __init__() """) end + # For calling Python with UTF-8 IO + function python_cmd(args) + env = copy(ENV) + env["PYTHONIOENCODING"] = "UTF-8" + setenv(`$(CONFIG.exepath) $args`, env) + end + # Find Python library libpath = something(CONFIG.libpath, get(ENV, "PYTHONJL_LIB", nothing), Some(nothing)) if libpath !== nothing @@ -113,78 +114,80 @@ function __init__() # Start the interpreter and register exit hooks C.Py_InitializeEx(0) CONFIG.isinitialized = true - check(C.Py_AtExit(@cfunction(()->(CONFIG.isinitialized = false; nothing), Cvoid, ()))) - atexit() do - CONFIG.isinitialized = false - check(C.Py_FinalizeEx()) - end - - # Some modules expect sys.argv to be set - pysysmodule.argv = pylist([""; ARGS]) - - # Some modules test for interactivity by checking if sys.ps1 exists - if isinteractive() && !pyhasattr(pysysmodule, "ps1") - pysysmodule.ps1 = ">>> " - end + # check(C.Py_AtExit(@cfunction(()->(CONFIG.isinitialized = false; nothing), Cvoid, ()))) + # atexit() do + # CONFIG.isinitialized = false + # check(C.Py_FinalizeEx()) + # end end end - with_gil() do - - # Is this the same Python as in Conda? - if !CONFIG.isconda && - haskey(ENV, "CONDA_PREFIX") && isdir(ENV["CONDA_PREFIX"]) && - haskey(ENV, "CONDA_PYTHON_EXE") && isfile(ENV["CONDA_PYTHON_EXE"]) && - realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? pyconvert(String, pysysmodule.executable) : CONFIG.exepath) - - CONFIG.isconda = true - CONFIG.condaenv = ENV["CONDA_PREFIX"] - CONFIG.exepath === nothing && (CONFIG.exepath = pyconvert(String, pysysmodule.executable)) - end - - # Get the python version - CONFIG.version = let (a,b,c,d,e) = pyconvert(Tuple{Int,Int,Int,String,Int}, pysysmodule.version_info) - VersionNumber(a, b, c, (d,), (e,)) - end - v"2" < CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version.major).$(CONFIG.version.minor) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") - - # EXPERIMENTAL: hooks to perform actions when certain modules are loaded - if !CONFIG.isembedded - py""" - import sys - class JuliaCompatHooks: - def __init__(self): - self.hooks = {} - def find_module(self, name, path=None): - hs = self.hooks.get(name) - if hs is not None: - for h in hs: - h() - def add_hook(self, name, h): - if name not in self.hooks: - self.hooks[name] = [h] - else: - self.hooks[name].append(h) - if name in sys.modules: - h() - JULIA_COMPAT_HOOKS = JuliaCompatHooks() - sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) - - # Before Qt is loaded, fix the path used to look up its plugins - qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) - JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) - JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) - JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) - JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) - """ - - @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin - IJulia.push_postexecute_hook() do - if CONFIG.pyplotautoshow && "matplotlib.pyplot" in pysysmodule.modules - pyplotshow() - end - end - end - end - end + # with_gil() do + + # if !CONFIG.isembedded + # # Some modules expect sys.argv to be set + # pysysmodule.argv = pylist([""; ARGS]) + + # # Some modules test for interactivity by checking if sys.ps1 exists + # if isinteractive() && !pyhasattr(pysysmodule, "ps1") + # pysysmodule.ps1 = ">>> " + # end + # end + + # # Is this the same Python as in Conda? + # if !CONFIG.isconda && + # haskey(ENV, "CONDA_PREFIX") && isdir(ENV["CONDA_PREFIX"]) && + # haskey(ENV, "CONDA_PYTHON_EXE") && isfile(ENV["CONDA_PYTHON_EXE"]) && + # realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? pyconvert(String, pysysmodule.executable) : CONFIG.exepath) + + # CONFIG.isconda = true + # CONFIG.condaenv = ENV["CONDA_PREFIX"] + # CONFIG.exepath === nothing && (CONFIG.exepath = pyconvert(String, pysysmodule.executable)) + # end + + # # Get the python version + # CONFIG.version = let (a,b,c,d,e) = pyconvert(Tuple{Int,Int,Int,String,Int}, pysysmodule.version_info) + # VersionNumber(a, b, c, (d,), (e,)) + # end + # v"2" < CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version.major).$(CONFIG.version.minor) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") + + # # EXPERIMENTAL: hooks to perform actions when certain modules are loaded + # if !CONFIG.isembedded + # py""" + # import sys + # class JuliaCompatHooks: + # def __init__(self): + # self.hooks = {} + # def find_module(self, name, path=None): + # hs = self.hooks.get(name) + # if hs is not None: + # for h in hs: + # h() + # def add_hook(self, name, h): + # if name not in self.hooks: + # self.hooks[name] = [h] + # else: + # self.hooks[name].append(h) + # if name in sys.modules: + # h() + # JULIA_COMPAT_HOOKS = JuliaCompatHooks() + # sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) + + # # Before Qt is loaded, fix the path used to look up its plugins + # qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) + # JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) + # """ + + # @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin + # IJulia.push_postexecute_hook() do + # if CONFIG.pyplotautoshow && "matplotlib.pyplot" in pysysmodule.modules + # pyplotshow() + # end + # end + # end + # end + # end end diff --git a/src/PyArray.jl b/src/old/PyArray.jl similarity index 100% rename from src/PyArray.jl rename to src/old/PyArray.jl diff --git a/src/PyBuffer.jl b/src/old/PyBuffer.jl similarity index 100% rename from src/PyBuffer.jl rename to src/old/PyBuffer.jl diff --git a/src/old/PyDict.jl b/src/old/PyDict.jl new file mode 100644 index 00000000..370d882d --- /dev/null +++ b/src/old/PyDict.jl @@ -0,0 +1,61 @@ +""" + PyDict{K=PyObject, V=PyObject}(o=pydict()) + +Wrap the Python dictionary `o` (or anything satisfying the mapping interface) as a Julia dictionary with keys of type `K` and values of type `V`. +""" +struct PyDict{K,V} <: AbstractDict{K,V} + o :: PyObject + PyDict{K,V}(o::PyObject) where {K,V} = new{K,V}(o) +end +PyDict{K,V}(o=pydict()) where {K,V} = PyDict{K,V}(pydict(o)) +PyDict{K}(o=pydict()) where {K} = PyDict{K,PyObject}(o) +PyDict(o=pydict()) = PyDict{PyObject}(o) +export PyDict + +pyobject(x::PyDict) = x.o + +function Base.iterate(x::PyDict{K,V}, it=pyiter(x.o.items())) where {K,V} + ptr = C.PyIter_Next(it) + if ptr == C_NULL + pyerrcheck() + nothing + else + kv = pynewobject(ptr) + (pyconvert(K, kv[0]) => pyconvert(V, kv[1])), it + end +end + +Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = + (pysetitem(x.o, convert(K, k), convert(V, v)); x) + +Base.getindex(x::PyDict{K,V}, k) where {K,V} = + pyconvert(V, pygetitem(x.o, convert(K, k))) + +Base.delete!(x::PyDict{K,V}, k) where {K,V} = + (pydelitem(x.o, convert(K, k)); x) + +Base.length(x::PyDict) = Int(pylen(x.o)) + +Base.empty!(x::PyDict) = (x.o.clear(); x) + +Base.copy(x::PyDict) = typeof(x)(x.o.copy()) + +function Base.get(x::PyDict{K,V}, _k, d) where {K,V} + k = convert(K, _k) + pycontains(x.o, k) ? x[k] : d +end + +function Base.get(f::Function, x::PyDict{K,V}, _k) where {K,V} + k = convert(K, _k) + pycontains(x.o, k) ? x[k] : f() +end + +function Base.get!(x::PyDict{K,V}, _k, d) where {K,V} + k = convert(K, _k) + pycontains(x.o, k) ? x[k] : (x[k] = d) +end + +function Base.get!(f::Function, x::PyDict{K,V}, _k) where {K,V} + k = convert(K, _k) + pycontains(x.o, k) ? x[k] : (x[k] = f()) +end diff --git a/src/PyIO.jl b/src/old/PyIO.jl similarity index 100% rename from src/PyIO.jl rename to src/old/PyIO.jl diff --git a/src/PyIterable.jl b/src/old/PyIterable.jl similarity index 100% rename from src/PyIterable.jl rename to src/old/PyIterable.jl diff --git a/src/PyList.jl b/src/old/PyList.jl similarity index 100% rename from src/PyList.jl rename to src/old/PyList.jl diff --git a/src/PyObjectArray.jl b/src/old/PyObjectArray.jl similarity index 100% rename from src/PyObjectArray.jl rename to src/old/PyObjectArray.jl diff --git a/src/PySet.jl b/src/old/PySet.jl similarity index 100% rename from src/PySet.jl rename to src/old/PySet.jl diff --git a/src/base.jl b/src/old/base.jl similarity index 100% rename from src/base.jl rename to src/old/base.jl diff --git a/src/bool.jl b/src/old/bool.jl similarity index 100% rename from src/bool.jl rename to src/old/bool.jl diff --git a/src/builtins.jl b/src/old/builtins.jl similarity index 100% rename from src/builtins.jl rename to src/old/builtins.jl diff --git a/src/bytearray.jl b/src/old/bytearray.jl similarity index 100% rename from src/bytearray.jl rename to src/old/bytearray.jl diff --git a/src/bytes.jl b/src/old/bytes.jl similarity index 100% rename from src/bytes.jl rename to src/old/bytes.jl diff --git a/src/collections.jl b/src/old/collections.jl similarity index 100% rename from src/collections.jl rename to src/old/collections.jl diff --git a/src/complex.jl b/src/old/complex.jl similarity index 100% rename from src/complex.jl rename to src/old/complex.jl diff --git a/src/convert.jl b/src/old/convert.jl similarity index 100% rename from src/convert.jl rename to src/old/convert.jl diff --git a/src/datetime.jl b/src/old/datetime.jl similarity index 100% rename from src/datetime.jl rename to src/old/datetime.jl diff --git a/src/dict.jl b/src/old/dict.jl similarity index 100% rename from src/dict.jl rename to src/old/dict.jl diff --git a/src/error.jl b/src/old/error.jl similarity index 100% rename from src/error.jl rename to src/old/error.jl diff --git a/src/old/eval.jl b/src/old/eval.jl new file mode 100644 index 00000000..0e28a9bc --- /dev/null +++ b/src/old/eval.jl @@ -0,0 +1,137 @@ +const SCOPES = Dict{Any,PyObject}() +const COMPILECACHE = Dict{Tuple{String,String,String},PyObject}() + +scope(s) = get!(pydict, SCOPES, s) +scope(s::PyObject) = s + +""" + pyeval(src, scope, locals=nothing) + +Evaluate Python expression `src` in the context of the given `scope` and `locals`. Return the value. + +If the `scope` is a Python object, it must be a `dict` to use as globals. + +Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyeval(src, @__MODULE__)` evaluates in a scope unique to the current module. +""" +pyeval(src, globals, locals=nothing) = pyevalfunc(src, scope(globals), locals) + +""" + pyexec(src, scope, locals=nothing) + +Execute Python expression `src` in the context of the given `scope` and `locals`. + +If the `scope` is a Python object, it must be a `dict` to use as globals. + +Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyexec(src, @__MODULE__)` executes in a scope unique to the current module. +""" +pyexec(src, globals, locals=nothing) = (pyexecfunc(src, scope(globals), locals); nothing) + +""" + pyevaldb(src, scope, locals=nothing) + +Same as `pyeval(src, scope, locals)` but evaluated inside a `pdb` debugger. +""" +pyevaldb(src, globals, locals=nothing) = pypdbmodule.runeval(src, scope(globals), locals) + +""" + pyexecdb(src, scope, locals=nothing) + +Same as `pyexec(src, scope, locals)` but evaluated inside a `pdb` debugger. +""" +pyexecdb(src, globals, locals=nothing) = (pypdbmodule.run(src, scope(globals), locals); nothing) + +""" + py"...."[flags] + +Evaluate (`v`) or execute (`x`) the given Python source code. + +Julia values may be interpolated into the source code with `\$` syntax. For a literal `\$`, enter `\$\$`. + +Execution occurs in a global scope unique to the current module. + +The flags can be any combination of the following characters: +- `v` (evaluate): Evaluate a single expression and return its value. +- `x` (execute): Execute the code and return `nothing`. +- `g` (globals): Return the dict of globals for the scope instead. +- `l` (locals): Perform the computation in a new local scope and return that scope. +- `c` (compile): Cache a compiled version of the source and re-use it each time, for speed. +- `d` (debug): Run inside a `pdb` debugger. + +If neither `v` nor `x` is specified and the code is a single line then `v` (evaluate) is assumed, otherwise `x` (execute). +""" +macro py_str(src::String, flags::String="") + # parse the flags + exec = '\n' in src + retglobals = false + retlocals = false + lazy = false + compile = false + debug = false + for f in flags + if f == 'x' + exec = true + elseif f == 'v' + exec = false + elseif f == 'g' + retglobals = true + elseif f == 'l' + retlocals = true + # elseif f == 'z' + # lazy = true + elseif f == 'c' + compile = true + elseif f == 'd' + debug = true + else + error("invalid flags: `py\"...\"$flags`") + end + end + # parse src for $-interpolations + chunks = String[] + interps = [] + i = firstindex(src) + while true + j = findnext('$', src, i) + if j === nothing + push!(chunks, src[i:end]) + break + else + push!(chunks, src[i:prevind(src,j)]) + end + if checkbounds(Bool, src, j+1) && src[j+1] == '$' + push!(chunks, "\$") + i = j+2 + else + ex, i = Meta.parse(src, j+1, greedy=false) + var = "_jl_interp_$(length(interps)+1)_" + push!(interps, var => ex) + push!(chunks, "($var)") + end + end + newsrcstr = newsrc = join(chunks) + # compile the code so there is string information + cfile = "julia:$(__source__.file):$(__source__.line)" + cmode = exec ? "exec" : "eval" + newsrc = :(pycompile($newsrcstr, $cfile, $cmode)) + if compile + # compile the code lazily (so the python parser is only invoked once) + # Julia crashes if you try to put pylazyobject(()->pycompile(...)) directly in the syntax tree. Is this a bug?? + # Instead, we use a run-time lookup table. + newsrc = :(get!(()->$newsrc, COMPILECACHE, ($newsrcstr, $cfile, $cmode))) + end + # make the expression + ex = :(let + globals = scope(@__MODULE__) + locals = $(retlocals ? :(pydict()) : :globals) + $([:(check(C.PyDict_SetItemString(globals, $k, pyobject($(esc(v)))))) for (k,v) in interps]...) + result = $(debug ? (exec ? :pyexecdb : :pyevaldb) : (exec ? :pyexec : :pyeval))($newsrc, globals, locals) + $([:(check(C.PyDict_DelItemString(globals, $k))) for (k,v) in interps]...) + $(retlocals ? :locals : retglobals ? :globals : exec ? nothing : :result) + end) + if lazy + # wrap as a lazy object + ex = :(pylazyobject(() -> $ex)) + end + ex +end +export @py_str diff --git a/src/float.jl b/src/old/float.jl similarity index 100% rename from src/float.jl rename to src/old/float.jl diff --git a/src/fraction.jl b/src/old/fraction.jl similarity index 100% rename from src/fraction.jl rename to src/old/fraction.jl diff --git a/src/function.jl b/src/old/function.jl similarity index 100% rename from src/function.jl rename to src/old/function.jl diff --git a/src/gui.jl b/src/old/gui.jl similarity index 100% rename from src/gui.jl rename to src/old/gui.jl diff --git a/src/import.jl b/src/old/import.jl similarity index 100% rename from src/import.jl rename to src/old/import.jl diff --git a/src/int.jl b/src/old/int.jl similarity index 100% rename from src/int.jl rename to src/old/int.jl diff --git a/src/io.jl b/src/old/io.jl similarity index 100% rename from src/io.jl rename to src/old/io.jl diff --git a/src/julia.jl b/src/old/julia.jl similarity index 100% rename from src/julia.jl rename to src/old/julia.jl diff --git a/src/list.jl b/src/old/list.jl similarity index 100% rename from src/list.jl rename to src/old/list.jl diff --git a/src/matplotlib.jl b/src/old/matplotlib.jl similarity index 100% rename from src/matplotlib.jl rename to src/old/matplotlib.jl diff --git a/src/newtype.jl b/src/old/newtype.jl similarity index 100% rename from src/newtype.jl rename to src/old/newtype.jl diff --git a/src/none.jl b/src/old/none.jl similarity index 100% rename from src/none.jl rename to src/old/none.jl diff --git a/src/number.jl b/src/old/number.jl similarity index 100% rename from src/number.jl rename to src/old/number.jl diff --git a/src/numpy.jl b/src/old/numpy.jl similarity index 100% rename from src/numpy.jl rename to src/old/numpy.jl diff --git a/src/object.jl b/src/old/object.jl similarity index 100% rename from src/object.jl rename to src/old/object.jl diff --git a/src/pandas.jl b/src/old/pandas.jl similarity index 100% rename from src/pandas.jl rename to src/old/pandas.jl diff --git a/src/pywith.jl b/src/old/pywith.jl similarity index 100% rename from src/pywith.jl rename to src/old/pywith.jl diff --git a/src/range.jl b/src/old/range.jl similarity index 100% rename from src/range.jl rename to src/old/range.jl diff --git a/src/sequence.jl b/src/old/sequence.jl similarity index 100% rename from src/sequence.jl rename to src/old/sequence.jl diff --git a/src/set.jl b/src/old/set.jl similarity index 100% rename from src/set.jl rename to src/old/set.jl diff --git a/src/slice.jl b/src/old/slice.jl similarity index 100% rename from src/slice.jl rename to src/old/slice.jl diff --git a/src/stdlib.jl b/src/old/stdlib.jl similarity index 100% rename from src/stdlib.jl rename to src/old/stdlib.jl diff --git a/src/str.jl b/src/old/str.jl similarity index 100% rename from src/str.jl rename to src/old/str.jl diff --git a/src/tuple.jl b/src/old/tuple.jl similarity index 100% rename from src/tuple.jl rename to src/old/tuple.jl diff --git a/src/type.jl b/src/old/type.jl similarity index 100% rename from src/type.jl rename to src/old/type.jl diff --git a/src/utils.jl b/src/utils.jl index 435ca3e8..2cd4c113 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -101,3 +101,7 @@ function pointer_from_obj(o::T) where {T} end p, c end + +isnull(p::Ptr) = Ptr{Cvoid}(p) == C_NULL +isnull(p::UnsafePtr) = isnull(pointer(p)) +ism1(x::T) where {T<:Number} = x == (zero(T)-one(T)) From 43794a855b9c1e619d60fdb723b5111628d4a655 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 14 Dec 2020 16:44:37 +0000 Subject: [PATCH 04/14] more conversions; adds @py, @pyv, @py_cmd, @pyv_cmd, @py_str --- src/Python.jl | 2 +- src/cpython/CPython.jl | 285 ++++++++++++++++++++++++++++++++++------- src/eval.jl | 169 +++++++++++++++++++++--- 3 files changed, 387 insertions(+), 69 deletions(-) diff --git a/src/Python.jl b/src/Python.jl index 235d1132..a23d2d24 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -56,7 +56,7 @@ const CONFIG = Config() struct PYERR end # Used to signal that conversion failed -struct PYUNCONVERTED end +struct NOTIMPLEMENTED end # C API include("cpython/CPython.jl") diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index bb61e102..bf93fcd4 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -1,10 +1,20 @@ module CPython using Libdl -using ..Python: CONFIG, isnull, ism1, PYERR, PYUNCONVERTED +using ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect using Base: @kwdef using UnsafePointers: UnsafePtr +tryconvert(::Type{T}, x::PYERR) where {T} = PYERR() +tryconvert(::Type{T}, x::NOTIMPLEMENTED) where {T} = NOTIMPLEMENTED() +tryconvert(::Type{T}, x::T) where {T} = x +tryconvert(::Type{T}, x) where {T} = + try + convert(T, x) + catch + NOTIMPLEMENTED() + end + @enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 const Py_single_input = 256 @@ -403,8 +413,25 @@ end ### EXCEPTIONS -const PyExc_TypeError__ref = Ref(PyPtr()) -PyExc_TypeError() = pyloadglobal(PyExc_TypeError__ref, :PyExc_TypeError) +for x in [:BaseException, :Exception, :StopIteration, :GeneratorExit, :ArithmeticError, + :LookupError, :AssertionError, :AttributeError, :BufferError, :EOFError, + :FloatingPointError, :OSError, :ImportError, :IndexError, :KeyError, :KeyboardInterrupt, + :MemoryError, :NameError, :OverflowError, :RuntimeError, :RecursionError, + :NotImplementedError, :SyntaxError, :IndentationError, :TabError, :ReferenceError, + :SystemError, :SystemExit, :TypeError, :UnboundLocalError, :UnicodeError, + :UnicodeEncodeError, :UnicodeDecodeError, :UnicodeTranslateError, :ValueError, + :ZeroDivisionError, :BlockingIOError, :BrokenPipeError, :ChildProcessError, + :ConnectionError, :ConnectionAbortedError, :ConnectionRefusedError, :FileExistsError, + :FileNotFoundError, :InterruptedError, :IsADirectoryError, :NotADirectoryError, + :PermissionError, :ProcessLookupError, :TimeoutError, :EnvironmentError, :IOError, + :WindowsError, :Warning, :UserWarning, :DeprecationWarning, :PendingDeprecationWarning, + :SyntaxWarning, :RuntimeWarning, :FutureWarning, :ImportWarning, :UnicodeWarning, + :BytesWarning, :ResourceWarning] + f = Symbol(:PyExc_, x) + r = Symbol(f, :__ref) + @eval const $r = Ref(PyPtr()) + @eval $f() = pyloadglobal($r, $(QuoteNode(f))) +end ### NONE @@ -421,7 +448,7 @@ PyNone_As(o, ::Type{T}) where {T} = elseif Missing <: T missing else - PYUNCONVERTED() + NOTIMPLEMENTED() end ### OBJECT @@ -492,6 +519,8 @@ end PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) PyObject_From(x::Nothing) = PyNone_New() PyObject_From(x::Bool) = PyBool_From(x) +PyObject_From(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = PyLong_From(x) +PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) PyObject_From(x::Tuple) = PyTuple_From(x) function PyObject_From(x) @@ -516,8 +545,8 @@ end PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) const PYOBJECT_AS_RULES = Dict{PyPtr, Union{Nothing,Function}}() -const PYOBJECT_AS_CRULES = Dict{Type, Dict{PyPtr, Ptr{Cvoid}}}() -const PYOBJECT_AS_CRULES_CACHE = Dict{Type, Dict{PyPtr, Any}}() +const PYOBJECT_AS_CRULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() +const PYOBJECT_AS_CRULES_CACHE = IdDict{Type, Dict{PyPtr, Any}}() function PyObject_As__rule(t::PyPtr) name = Py_DecRef(PyObject_GetAttrString(t, "__name__")) do tnameo @@ -530,62 +559,66 @@ function PyObject_As__rule(t::PyPtr) end end name == PYERR() && return PYERR() - @show name PyObject_As__rule(t, Val(Symbol(name))) end -PyObject_As__rule(::PyPtr, ::Val) = nothing -PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.NoneType")}) = PyNone_As -PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.bool")}) = PyBool_As -PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.str")}) = PyUnicode_As -PyObject_As__rule(::PyPtr, ::Val{Symbol("builtins.bytes")}) = PyBytes_As +PyObject_As__rule(t::PyPtr, ::Val) = nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.NoneType")}) = Py_Is(t, Py_Type(Py_None())) ? PyNone_As : nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.bool")}) = Py_Is(t, PyBool_Type()) ? PyBool_As : nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.str")}) = Py_Is(t, PyUnicode_Type()) ? PyUnicode_As : nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.bytes")}) = Py_Is(t, PyBytes_Type()) ? PyBytes_As : nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.int")}) = Py_Is(t, PyLong_Type()) ? PyLong_As : nothing +PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.float")}) = Py_Is(t, PyFloat_Type()) ? PyFloat_As : nothing struct PyObject_As__crule_struct{T,F} f :: F end -function (r::PyObject_As__crule_struct{T,F})(o::PyPtr, ptr::Ptr) where {T,F} +function (r::PyObject_As__crule_struct{T,F})(o::PyPtr, ref::Base.RefValue{Any}) where {T,F} res = r.f(o, T) if res === PYERR() return Cint(2) - elseif res === PYUNCONVERTED() + elseif res === NOTIMPLEMENTED() return Cint(1) else - unsafe_store!(ptr, res) + ref[] = res return Cint(0) end end function PyObject_As(o::PyPtr, ::Type{T}) where {T} - # MRO + # Run through the MRO, applying conversion rules that depend on the supertypes of the type of o. + # For speed, the conversion functions are cached as C function pointers. + # These take as inputs the PyPtr o and a Ptr{Any} in which to store the result. + # They return 0 on success, 1 on no-conversion and 2 on error. mro = PyType_MRO(Py_Type(o)) ref = Ref{Any}() rules = get!(Dict{PyPtr, Ptr{Cvoid}}, PYOBJECT_AS_CRULES, T)::Dict{PyPtr,Ptr{Cvoid}} for i in 1:PyTuple_Size(mro) t = PyTuple_GetItem(mro, i-1) isnull(t) && return PYERR() - crule = get(rules, t, nothing) - if crule === nothing + crule = get(rules, t, missing) + if crule === missing rule = PyObject_As__rule(t) rule === PYERR() && return PYERR() if rule === nothing rules[t] = C_NULL continue else - crulefunc = @cfunction($(PyObject_As__crule_struct{T, typeof(rule)}(rule)), Cint, (PyPtr, Ptr{Any})) + crulefunc = @cfunction($(PyObject_As__crule_struct{T, typeof(rule)}(rule)), Cint, (PyPtr, Any)) get!(Dict{PyPtr, Any}, PYOBJECT_AS_CRULES_CACHE, T)[t] = crulefunc crule = rules[t] = Base.unsafe_convert(Ptr{Cvoid}, crulefunc) end elseif crule == C_NULL continue end - res = ccall(crule, Cint, (PyPtr, Ptr{Any}), o, ref) + res = ccall(crule, Cint, (PyPtr, Any), o, ref) if res == 0 return ref[]::T elseif res == 2 return PYERR() end end - PYUNCONVERTED() + NOTIMPLEMENTED() end PyObject_As(o, ::Type{T}) where {T} = GC.@preserve o PyObject_As(Base.unsafe_convert(PyPtr, o), T) @@ -601,6 +634,31 @@ PyObject_As(o, ::Type{T}) where {T} = GC.@preserve o PyObject_As(Base.unsafe_con @cdef :PyMapping_SetItemString Cint (PyPtr, Cstring, PyPtr) @cdef :PyMapping_GetItemString PyPtr (PyPtr, Cstring) +PyMapping_ExtractOneAs(o, k, ::Type{T}) where {T} = + Py_DecRef(PyMapping_GetItemString(o, string(k))) do x + v = PyObject_As(x, T) + if v === NOTIMPLEMENTED() + PyErr_SetString(PyExc_TypeError(), "Cannot convert this '$(PyType_Name(Py_Type(x)))' at key '$k' to a Julia '$T'") + PYERR() + else + v + end + end + +PyMapping_ExtractAs(o::PyPtr, ::Type{NamedTuple{names,types}}) where {names, types} = begin + t = PyMapping_ExtractAs(o, names, types) + t === PYERR() ? PYERR() : NamedTuple{names,types}(t) +end +PyMapping_ExtractAs(o::PyPtr, names::Tuple, ::Type{types}) where {types<:Tuple} = begin + v = PyMapping_ExtractOneAs(o, first(names), Base.tuple_type_head(types)) + v === PYERR() && return PYERR() + vs = PyMapping_ExtractAs(o::PyPtr, Base.tail(names), Base.tuple_type_tail(types)) + vs === PYERR() && return PYERR() + (v, vs...) +end +PyMapping_ExtractAs(o::PyPtr, names::Tuple{}, ::Type{Tuple{}}) = () +PyMapping_ExtractAs(o, ::Type{T}) where {T} = GC.@preserve o PyMapping_ExtractAs(Base.unsafe_convert(PyPtr, o), T) + ### METHOD @cdef :PyInstanceMethod_New PyPtr (PyPtr,) @@ -610,8 +668,8 @@ PyObject_As(o, ::Type{T}) where {T} = GC.@preserve o PyObject_As(Base.unsafe_con @cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) @cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) -const PyUnicode_Type__ref = Ref{PyPtr}() -PyUnicode_Type() = pyloadglobal(PyUnicode_Type__ref, :PyUnicode_Type) +const PyUnicode_Type__ref = Ref(C_NULL) +PyUnicode_Type() = PyPtr(pyglobal(PyUnicode_Type__ref, :PyUnicode_Type)) PyUnicode_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_UNICODE_SUBCLASS) PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) @@ -619,14 +677,36 @@ PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) -PyUnicode_As(o, ::Type) = PYUNCONVERTED() - -function PyUnicode_As(o, ::Type{T}) where {T<:Union{String,Vector{Int8},Vector{UInt8}}} - b = PyUnicode_AsUTF8String(o) - isnull(b) && return PYERR() - r = PyBytes_As(b, T) - Py_DecRef(b) - r +PyUnicode_As(o, ::Type{T}) where {T} = begin + if (S = _typeintersect(T, AbstractString)) != Union{} + r = Py_DecRef(PyUnicode_AsUTF8String(o)) do b + PyBytes_As(b, S) + end + r === NOTIMPLEMENTED() || return r + end + if Symbol <: T + s = PyUnicode_As(o, String) + s === PYERR() && return PYERR() + s isa String && return Symbol(s) + end + if (S = _typeintersect(T, AbstractChar)) != Union{} + s = PyUnicode_As(o, String) + s === PYERR() && return PYERR() + if s isa String + if length(s) == 1 + c = first(s) + r = tryconvert(S, c) + r === NOTIMPLEMENTED() || return r + end + end + end + if (S = _typeintersect(T, AbstractVector)) != Union{} + r = Py_DecRef(PyUnicode_AsUTF8String(o)) do b + PyBytes_As(b, S) + end + r === NOTIMPLEMENTED() || return r + end + NOTIMPLEMENTED() end ### BYTES @@ -637,22 +717,35 @@ end PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = PyBytes_FromStringAndSize(pointer(s), sizeof(s)) -PyBytes_As(o, ::Type) = PYUNCONVERTED() - -function PyBytes_As(o, ::Type{String}) - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return PYERR() - Base.unsafe_string(ptr[], len[]) -end - -function PyBytes_As(o, ::Type{Vector{T}}) where {T<:Union{UInt8,Int8}} - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return PYERR() - copy(Base.unsafe_wrap(Vector{T}, Ptr{T}(ptr[]), len[])) +PyBytes_As(o, ::Type{T}) where {T} = begin + if (S = _typeintersect(T, AbstractVector{UInt8})) != Union{} + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return PYERR() + v = copy(Base.unsafe_wrap(Vector{UInt8}, Ptr{UInt8}(ptr[]), len[])) + r = tryconvert(S, v) + r === NOTIMPLEMENTED() || return r + end + if (S = _typeintersect(T, AbstractVector{Int8})) != Union{} + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return PYERR() + v = copy(Base.unsafe_wrap(Vector{Int8}, Ptr{Int8}(ptr[]), len[])) + r = tryconvert(S, v) + r === NOTIMPLEMENTED() || return r + end + if (S = _typeintersect(T, AbstractString)) != Union{} + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return PYERR() + s = Base.unsafe_string(ptr[], len[]) + r = tryconvert(S, s) + r === NOTIMPLEMENTED() || return r + end + NOTIMPLEMENTED() end ### TUPLE @@ -691,6 +784,13 @@ PyType_MRO(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) +const PyType_Type__ref = Ref(C_NULL) +PyType_Type() = PyPtr(pyglobal(PyType_Type__ref, :PyType_Type)) + +PyType_Check(o) = Py_TypeCheck(o, Py_TPFLAGS_TYPE_SUBCLASS) + +PyType_CheckExact(o) = Py_TypeCheckExact(o, PyType_Type()) + ### NUMBER @cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) @@ -745,7 +845,7 @@ Py_False() = PyPtr(pyglobal(Py_False__ref, :_Py_FalseStruct)) PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) -PyBool_Check(o) = Py_Type(o) == PyBool_Type() +PyBool_Check(o) = Py_TypeCheckExact(o, PyBool_Type()) PyBool_As(o, ::Type{T}) where {T} = if Bool <: T @@ -758,7 +858,7 @@ PyBool_As(o, ::Type{T}) where {T} = PYERR() end else - PYUNCONVERTED() + NOTIMPLEMENTED() end @@ -770,11 +870,98 @@ PyBool_As(o, ::Type{T}) where {T} = @cdef :PyLong_AsLongLong Clonglong (PyPtr,) @cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) +const PyLong_Type__ref = Ref(C_NULL) +PyLong_Type() = PyPtr(pyglobal(PyLong_Type__ref, :PyLong_Type)) + +PyLong_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LONG_SUBCLASS) + +PyLong_CheckExact(o) = Py_TypeCheckExact(o, PyLong_Type()) + +PyLong_From(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) = + if x isa Signed && typemin(Clonglong) ≤ x ≤ typemax(Clonglong) + PyLong_FromLongLong(x) + elseif typemin(Culonglong) ≤ x ≤ typemax(Culonglong) + PyLong_FromUnsignedLongLong(x) + else + PyLong_FromString(string(x), C_NULL, 10) + end + +PyLong_From(x::Integer) = begin + y = tryconvert(BigInt, x) + y === PYERR() && return PyPtr() + y === NOTIMPLEMENTED() && (PyErr_SetString(PyExc_NotImplementedError(), "Cannot convert this Julia '$(typeof(x))' to a Python 'int'"); return PyPtr()) + PyLong_From(y::BigInt) +end + +PyLong_As(o, ::Type{T}) where {T} = begin + if (S = _typeintersect(T, Integer)) != Union{} + # first try to convert to Clonglong (or Culonglong if unsigned) + x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) + if !ism1(x) || !PyErr_IsSet() + # success + return tryconvert(S, x) + elseif PyErr_IsSet(PyExc_OverflowError()) + # overflows Clonglong or Culonglong + PyErr_Clear() + if S in (Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128) && typemin(typeof(x)) ≤ typemin(S) && typemax(S) ≤ typemax(typeof(x)) + # definitely overflows S, give up now + return NOTIMPLEMENTED() + else + # try converting to String then BigInt then S + s = PyObject_StrAs(o, String) + s === PYERR() && return PYERR() + y = tryparse(BigInt, s) + y === nothing && (PyErr_SetString(PyExc_ValueError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia 'BigInt' because its string representation cannot be parsed as an integer"); return PYERR()) + return tryconvert(S, y::BigInt) + end + else + # other error + return PYERR() + end + elseif (S = _typeintersect(T, Real)) != Union{} + return tryconvert(S, PyLong_As(o, Integer)) + elseif (S = _typeintersect(T, Number)) != Union{} + return tryconvert(S, PyLong_As(o, Integer)) + else + return tryconvert(T, PyLong_As(o, Integer)) + end + NOTIMPLEMENTED() +end + ### FLOAT @cdef :PyFloat_FromDouble PyPtr (Cdouble,) @cdef :PyFloat_AsDouble Cdouble (PyPtr,) +const PyFloat_Type__ref = Ref(C_NULL) +PyFloat_Type() = PyPtr(pyglobal(PyFloat_Type__ref, :PyFloat_Type)) + +PyFloat_Check(o) = Py_TypeCheck(o, PyFloat_Type()) + +PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) + +PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) + +PyFloat_As(o, ::Type{T}) where {T} = begin + x = PyFloat_AsDouble(o) + ism1(x) && PyErr_IsSet() && return PYERR() + if Float64 <: T + return convert(Float64, x) + elseif Float32 <: T + return convert(Float32, x) + elseif Float16 <: T + return convert(Float16, x) + elseif (S = _typeintersect(T, AbstractFloat)) != Union{} + return tryconvert(S, x) + elseif (S = _typeintersect(T, Real)) != Union{} + return tryconvert(S, x) + elseif (S = _typeintersect(T, Number)) != Union{} + return tryconvert(S, x) + else + return tryconvert(T, x) + end +end + ### COMPLEX @cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) diff --git a/src/eval.jl b/src/eval.jl index 8780f57b..97f021d3 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,4 +1,4 @@ -function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Nothing}=globals, extras::Union{NamedTuple,Nothing}=nothing) where {R} +function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Nothing}=globals, extras::Union{NamedTuple,Nothing,Tuple{}}=nothing) where {R} # get code cptr = pyptr(co) # get globals & ensure __builtins__ is set @@ -18,7 +18,7 @@ function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Not lptr = pyptr(locals) end # insert extra locals - if extras !== nothing + if extras isa NamedTuple for (k,v) in pairs(extras) vo = C.PyObject_From(v) if isnull(vo) @@ -47,8 +47,11 @@ function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Not return nothing elseif R <: NamedTuple && isconcretetype(R) C.Py_DecRef(rptr) + ret = C.PyMapping_ExtractAs(lptr, R) locals===nothing && C.Py_DecRef(lptr) - error("returning NamedTuple not implemented yet") + ret === PYERR() && pythrow() + ret === NOTIMPLEMENTED() && pythrow() + return ret::R else C.Py_DecRef(rptr) locals===nothing && C.Py_DecRef(lptr) @@ -56,12 +59,12 @@ function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Not end elseif co.mode == :eval ret = C.PyObject_As(rptr, R) - ret === PYUNCONVERTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert this '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$R'") + ret === NOTIMPLEMENTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert this '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$R'") C.Py_DecRef(rptr) locals===nothing && C.Py_DecRef(lptr) ret === PYERR() && pythrow() - ret === PYUNCONVERTED() && pythrow() - return ret + ret === NOTIMPLEMENTED() && pythrow() + return ret::R else C.Py_DecRef(rptr) locals===nothing && C.Py_DecRef(lptr) @@ -70,20 +73,148 @@ function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Not end export pyeval -module CompiledCode end +module CompiledCode + import ..Python: PyCode + stash(co::PyCode) = begin + nm = gensym() + @eval $nm = $co + co + end + stash(args...) = stash(PyCode(args...)) +end -macro pyeval(R, code::String, locals=nothing) - co = PyCode(code, "", :eval) - nm = gensym() - Base.eval(CompiledCode, :($nm = $co)) - :(pyeval($(esc(R)), $co, $(esc(:pyglobals)), $(esc(locals)))) +pyeval_macro(src, mode, args...) = begin + # find the code argument + icode = findfirst(args) do x + x isa Expr && x.head == :macrocall && x.args[1] == :(`foo`).args[1] + end + icode in (1,2) || error() + code = args[icode].args[end] + # the return type + rettypearg = icode==2 ? args[1] : mode==:eval ? Any : Nothing + # parse the remaining arguments - the first one is optionally the locals, the rest are var=value pairs to include in the locals + kvs = [] + locals = nothing + for (i,arg) in enumerate(args[icode+1:end]) + if arg isa Expr && arg.head == :(=) && arg.args[1] isa Symbol + push!(kvs, arg.args[1] => arg.args[2]) + elseif i == 1 + locals = arg + else + error() + end + end + # find any interpolations + i = firstindex(code) + codechunks = String[] + interps = [] + chunk = "" + while true + j = findnext('$', code, i) + if j === nothing + push!(codechunks, chunk * code[i:end]) + break + elseif checkbounds(Bool, code, j+1) && code[j+1] == '$' + chunk *= code[i:j] + i = j+2 + else + push!(codechunks, chunk * code[i:j-1]) + chunk = "" + ex, i = Meta.parse(code, j+1, greedy=false) + push!(interps, ex) + end + end + # identify which are LHS and which are RHS + # currently only the pattern "^ $(...) =" is recognized as a LHS + # TODO: multiple assignment + # TODO: mutating assignment + islhs = [((match(r"\n\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"\s*=", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] + # do the interpolation + intvars = ["_jl_interp_$(i)_" for i in 1:length(interps)] + for (k,v,lhs) in zip(intvars, interps, islhs) + lhs || push!(kvs, Symbol(k) => v) + end + newcode = join([c * (checkbounds(Bool, intvars, i) ? islhs[i] ? intvars[i] : "($(intvars[i]))" : "") for (i,c) in enumerate(codechunks)]) + # for LHS interpolation, extract out type annotations + if any(islhs) + mode == :exec || error("interpolation on LHS only allowed in exec mode") + icode == 1 || error("cannot specify return type when interpolation on LHS is used") + rettypearg = :($NamedTuple{($([QuoteNode(Symbol(n)) for (n,lhs) in zip(intvars,islhs) if lhs]...),),Tuple{$([(ex isa Expr && ex.head == :(::)) ? ex.args[2] : Any for (ex,lhs) in zip(interps,islhs) if lhs]...)}}) + end + # make the code object + co = CompiledCode.stash(newcode, "", mode) + # call pyeval + ret = :(pyeval($(esc(rettypearg)), $(co), $(esc(:pyglobals)), $(esc(locals)), ($([:($k = $(esc(v))) for (k,v) in kvs]...),))) + # assign + if any(islhs) + ret = :(($([(ex isa Expr && ex.head == :(::)) ? esc(ex.args[1]) : esc(ex) for (ex,lhs) in zip(interps,islhs) if lhs]...),) = $ret; nothing) + end + ret end -export @pyeval -macro pyexec(code::String, locals=nothing) - co = PyCode(code, "", :exec) - nm = gensym() - Base.eval(CompiledCode, :($nm = $co)) - :(pyeval(Nothing, $co, $(esc(:pyglobals)), $(esc(locals)))) +""" + @py [rettype] `...` [locals] [var=val, ...] + +Executes the given Python code. + +Julia values can be interpolated using the usual `\$(...)` syntax. +Additionally, assignment to interpolations is supported: e.g. `\$(x::T) = ...` will convert the right hand side to a `T` and assign it to `x`. + +The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. + +The resulting expression has type `rettype`, which may be `Nothing` (the default) or `NamedTuple{names,types}` to extract variables with the given names as a named tuple. +""" +macro py(args...) + pyeval_macro(__source__, :exec, args...) +end +export @py + +""" + py"..." + +Shorthand for ```@py `...` ```. +""" +macro py_str(code::String) + pyeval_macro(__source__, :exec, Expr(:macrocall, :(`foo`).args[1], code)) +end +export @py_str + +""" + @pyv [rettype] `...` [locals] [var=val, ...] + +Evaluate the given Python code. + +Julia values can be interpolated using the usual `\$(...)` syntax. + +The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. + +The result is converted to a `rettype`, which defaults to `PyObject`. +""" +macro pyv(args...) + pyeval_macro(__source__, :eval, args...) +end +export @pyv + +""" + py`...` :: PyCode + +A Python code object in "exec" mode which is compiled only once. + +Suitable for using as the `code` argument to `pyeval`. +""" +macro py_cmd(code::String) + CompiledCode.stash(code, "", :exec) +end +export @py_cmd + +""" + pyv`...` :: PyCode + +A Python code object in "eval" mode which is compiled only once. + +Suitable for using as the `code` argument to `pyexec`. +""" +macro pyv_cmd(code::String) + CompiledCode.stash(code, "", :eval) end -export @pyexec +export @pyv_cmd From 20186f4379be076d9f1e205df356bc54dd1bda08 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 14 Dec 2020 17:50:29 +0000 Subject: [PATCH 05/14] remove pyeval() and puts all its logic straight into @py etc. (saves memory allocations) --- src/eval.jl | 143 +++++++++++++++++++++------------------------------- 1 file changed, 57 insertions(+), 86 deletions(-) diff --git a/src/eval.jl b/src/eval.jl index 97f021d3..6aef7f27 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,78 +1,3 @@ -function pyeval(::Type{R}, co::PyCode, globals::PyDict, locals::Union{PyDict,Nothing}=globals, extras::Union{NamedTuple,Nothing,Tuple{}}=nothing) where {R} - # get code - cptr = pyptr(co) - # get globals & ensure __builtins__ is set - gptr = pyptr(globals) - if !globals.hasbuiltins - if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 - err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) - ism1(err) && pythrow() - end - globals.hasbuiltins = true - end - # get locals (ALLOCATES lptr if locals===nothing) - if locals === nothing - lptr = C.PyDict_New() - isnull(lptr) && pythrow() - else - lptr = pyptr(locals) - end - # insert extra locals - if extras isa NamedTuple - for (k,v) in pairs(extras) - vo = C.PyObject_From(v) - if isnull(vo) - locals===nothing && C.Py_DecRef(lptr) - pythrow() - end - err = C.PyMapping_SetItemString(lptr, string(k), vo) - C.Py_DecRef(vo) - if ism1(err) - locals===nothing && C.Py_DecRef(lptr) - pythrow() - end - end - end - # Call eval (ALLOCATES rptr) - rptr = C.PyEval_EvalCode(cptr, gptr, lptr) - if isnull(rptr) - locals === nothing && C.Py_DecRef(lptr) - pythrow() - end - # TODO: convert rptr using PyObject_As - if co.mode == :exec - if R <: Nothing - C.Py_DecRef(rptr) - locals===nothing && C.Py_DecRef(lptr) - return nothing - elseif R <: NamedTuple && isconcretetype(R) - C.Py_DecRef(rptr) - ret = C.PyMapping_ExtractAs(lptr, R) - locals===nothing && C.Py_DecRef(lptr) - ret === PYERR() && pythrow() - ret === NOTIMPLEMENTED() && pythrow() - return ret::R - else - C.Py_DecRef(rptr) - locals===nothing && C.Py_DecRef(lptr) - error("invalid return type $(R)") - end - elseif co.mode == :eval - ret = C.PyObject_As(rptr, R) - ret === NOTIMPLEMENTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert this '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$R'") - C.Py_DecRef(rptr) - locals===nothing && C.Py_DecRef(lptr) - ret === PYERR() && pythrow() - ret === NOTIMPLEMENTED() && pythrow() - return ret::R - else - C.Py_DecRef(rptr) - locals===nothing && C.Py_DecRef(lptr) - error("invalid mode $(repr(co.mode))") - end -end -export pyeval - module CompiledCode import ..Python: PyCode stash(co::PyCode) = begin @@ -138,22 +63,70 @@ pyeval_macro(src, mode, args...) = begin # for LHS interpolation, extract out type annotations if any(islhs) mode == :exec || error("interpolation on LHS only allowed in exec mode") - icode == 1 || error("cannot specify return type when interpolation on LHS is used") - rettypearg = :($NamedTuple{($([QuoteNode(Symbol(n)) for (n,lhs) in zip(intvars,islhs) if lhs]...),),Tuple{$([(ex isa Expr && ex.head == :(::)) ? ex.args[2] : Any for (ex,lhs) in zip(interps,islhs) if lhs]...)}}) end # make the code object co = CompiledCode.stash(newcode, "", mode) - # call pyeval - ret = :(pyeval($(esc(rettypearg)), $(co), $(esc(:pyglobals)), $(esc(locals)), ($([:($k = $(esc(v))) for (k,v) in kvs]...),))) - # assign - if any(islhs) - ret = :(($([(ex isa Expr && ex.head == :(::)) ? esc(ex.args[1]) : esc(ex) for (ex,lhs) in zip(interps,islhs) if lhs]...),) = $ret; nothing) + # go + freelocals = locals === nothing ? :(GC.@preserve locals C.Py_DecRef(lptr)) : nothing + quote + # get the code pointer + cptr = pyptr($co) + # get the globals pointer + globals = $(esc(:pyglobals)) + gptr = pyptr(globals) + # ensure globals includes builtins + if !globals.hasbuiltins + if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 + err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) + ism1(err) && pythrow() + end + globals.hasbuiltins = true + end + # get locals (ALLOCATES lptr if locals===nothing) + $(locals === nothing ? :(lptr = C.PyDict_New(); isnull(lptr) && pythrow()) : :(locals = $(esc(locals)); lptr = pyptr(locals))) + # insert extra locals + $([:(let; vo=C.PyObject_From($(esc(v))); isnull(vo) && ($freelocals; pythrow()); err=C.PyMapping_SetItemString(lptr, $(string(k)), vo); C.Py_DecRef(vo); ism1(err) && ($freelocals; pythrow()); end) for (k,v) in kvs]...) + # Call eval (ALLOCATES rptr) + rptr = GC.@preserve globals C.PyEval_EvalCode(cptr, gptr, lptr) + isnull(rptr) && ($freelocals; pythrow()) + # extract values + $( + if mode == :eval + quote + $freelocals + res = C.PyObject_As(rptr, $(esc(rettypearg))) + C.Py_DecRef(rptr) + res == PYERR() && pythrow() + res == NOTIMPLEMENTED() && (C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert return value of type '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$($(esc(rettypearg)))'"); pythrow()) + return res::$(esc(rettypearg)) + end + elseif mode == :exec + quote + C.Py_DecRef(rptr) + $((((jv,jt) = (ex isa Expr && ex.head == :(::)) ? (ex.args[1], esc(ex.args[2])) : (ex, Any); quote + $(esc(jv)) = let + xo = C.PyMapping_GetItemString(lptr, $v) + isnull(xo) && ($freelocals; pythrow()) + x = C.PyObject_As(xo, $jt) + x===NOTIMPLEMENTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert return value '$($(string(jv)))' of type '$(C.PyType_Name(C.Py_Type(xo)))' to a Julia '$($jt)'") + C.Py_DecRef(xo) + x===PYERR() && ($freelocals; pythrow()) + x===NOTIMPLEMENTED() && ($freelocals; pythrow()) + x::$jt; + end + end) for (ex,v,lhs) in zip(interps,intvars,islhs) if lhs)...) + $freelocals + nothing + end + else + error() + end + ) end - ret end """ - @py [rettype] `...` [locals] [var=val, ...] + @py `...` [locals] [var=val, ...] Executes the given Python code. @@ -161,8 +134,6 @@ Julia values can be interpolated using the usual `\$(...)` syntax. Additionally, assignment to interpolations is supported: e.g. `\$(x::T) = ...` will convert the right hand side to a `T` and assign it to `x`. The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. - -The resulting expression has type `rettype`, which may be `Nothing` (the default) or `NamedTuple{names,types}` to extract variables with the given names as a named tuple. """ macro py(args...) pyeval_macro(__source__, :exec, args...) From b9656ce96b1dba2fa79f77c93498daeefe600c00 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Dec 2020 18:18:27 +0000 Subject: [PATCH 06/14] loadsa dev --- src/PyCode.jl | 34 +- src/PyDict.jl | 121 +++- src/PyException.jl | 168 +++--- src/PyInternedString.jl | 29 + src/PyList.jl | 75 +++ src/PyObject.jl | 132 +++++ src/PyRef.jl | 49 ++ src/PySet.jl | 83 +++ src/Python.jl | 34 +- src/builtins.jl | 628 ++++++++++++++++++++ src/cpython/CPython.jl | 1088 ++++------------------------------- src/cpython/bool.jl | 22 + src/cpython/buffer.jl | 28 + src/cpython/bytes.jl | 29 + src/cpython/collections.jl | 216 +++++++ src/cpython/complex.jl | 31 + src/cpython/consts.jl | 309 ++++++++++ src/cpython/dict.jl | 47 ++ src/cpython/error.jl | 0 src/cpython/float.jl | 23 + src/cpython/fundamentals.jl | 96 ++++ src/cpython/int.jl | 62 ++ src/cpython/iter.jl | 1 + src/cpython/list.jl | 3 + src/cpython/mapping.jl | 3 + src/cpython/method.jl | 1 + src/cpython/none.jl | 9 + src/cpython/number.jl | 34 ++ src/cpython/object.jl | 236 ++++++++ src/cpython/sequence.jl | 4 + src/cpython/set.jl | 2 + src/cpython/str.jl | 44 ++ src/cpython/tuple.jl | 16 + src/cpython/type.jl | 40 ++ src/eval.jl | 156 ++--- src/init.jl | 5 + src/old/PyList.jl | 43 -- src/old/PySet.jl | 73 --- src/utils.jl | 42 +- 39 files changed, 2693 insertions(+), 1323 deletions(-) create mode 100644 src/PyInternedString.jl create mode 100644 src/PyList.jl create mode 100644 src/PyObject.jl create mode 100644 src/PyRef.jl create mode 100644 src/PySet.jl create mode 100644 src/builtins.jl create mode 100644 src/cpython/bool.jl create mode 100644 src/cpython/buffer.jl create mode 100644 src/cpython/bytes.jl create mode 100644 src/cpython/collections.jl create mode 100644 src/cpython/complex.jl create mode 100644 src/cpython/consts.jl create mode 100644 src/cpython/dict.jl create mode 100644 src/cpython/error.jl create mode 100644 src/cpython/float.jl create mode 100644 src/cpython/fundamentals.jl create mode 100644 src/cpython/int.jl create mode 100644 src/cpython/iter.jl create mode 100644 src/cpython/list.jl create mode 100644 src/cpython/mapping.jl create mode 100644 src/cpython/method.jl create mode 100644 src/cpython/none.jl create mode 100644 src/cpython/number.jl create mode 100644 src/cpython/object.jl create mode 100644 src/cpython/sequence.jl create mode 100644 src/cpython/set.jl create mode 100644 src/cpython/str.jl create mode 100644 src/cpython/tuple.jl create mode 100644 src/cpython/type.jl delete mode 100644 src/old/PyList.jl delete mode 100644 src/old/PySet.jl diff --git a/src/PyCode.jl b/src/PyCode.jl index 705bc7ad..3e426aee 100644 --- a/src/PyCode.jl +++ b/src/PyCode.jl @@ -1,37 +1,21 @@ mutable struct PyCode - ptr :: CPyPtr + ref :: PyRef code :: String filename :: String mode :: Symbol - function PyCode(code::String, filename::String, mode::Symbol) + PyCode(code::String, filename::String, mode::Symbol) = begin mode in (:exec, :eval) || error("invalid mode $(repr(mode))") - o = new(CPyPtr(), code, filename, mode) - finalizer(o) do o - if CONFIG.isinitialized - ptr = getfield(o, :ptr) - if !isnull(ptr) - with_gil(false) do - C.Py_DecRef(ptr) - end - end - end - end - return o + new(PyRef(), code, filename, mode) end end export PyCode -function pyptr(co::PyCode) - ptr = getfield(co, :ptr) +ispyreftype(::Type{PyCode}) = true +pyptr(co::PyCode) = begin + ptr = co.ref.ptr if isnull(ptr) - ptr = C.Py_CompileString(co.code, co.filename, co.mode == :exec ? C.Py_file_input : co.mode == :eval ? C.Py_eval_input : error("invalid mode $(repr(co.mode))")) - if isnull(ptr) - pythrow() - else - setfield!(co, :ptr, ptr) - ptr - end - else - ptr + ptr = co.ref.ptr = C.Py_CompileString(co.code, co.filename, co.mode == :exec ? C.Py_file_input : co.mode == :eval ? C.Py_eval_input : error("invalid mode $(repr(co.mode))")) end + ptr end +Base.unsafe_convert(::Type{CPyPtr}, x::PyCode) = checknull(pyptr(x)) diff --git a/src/PyDict.jl b/src/PyDict.jl index d02de681..3b12f1cd 100644 --- a/src/PyDict.jl +++ b/src/PyDict.jl @@ -1,37 +1,104 @@ -mutable struct PyDict - ptr :: CPyPtr +""" + PyDict{K=PyObject, V=PyObject}([o]) + +Wrap the Python dictionary `o` (or anything satisfying the mapping interface) as a Julia dictionary with keys of type `K` and values of type `V`. + +If `o` is not given, an empty dict is created. +""" +mutable struct PyDict{K,V} <: AbstractDict{K,V} + ref :: PyRef hasbuiltins :: Bool - function PyDict() - o = new(CPyPtr(), false) - finalizer(o) do o - if CONFIG.isinitialized - ptr = getfield(o, :ptr) - if !isnull(ptr) - with_gil(false) do - C.Py_DecRef(ptr) - end - end - end - end - return o - end + PyDict{K,V}(o) where {K,V} = new(PyRef(o), false) + PyDict{K,V}() where {K,V} = new(PyRef(), false) end +PyDict{K}(args...) where {K} = PyDict{K,PyObject}(args...) +PyDict(args...) = PyDict{PyObject,PyObject}(args...) export PyDict -const pyglobals = PyDict() +const pyglobals = PyDict{String}() export pyglobals -function pyptr(x::PyDict) - ptr = getfield(x, :ptr) +ispyreftype(::Type{<:PyDict}) = true +pyptr(x::PyDict) = begin + ptr = x.ref.ptr if isnull(ptr) - ptr = C.PyDict_New() - if isnull(ptr) - pythrow() - else - setfield!(x, :ptr, ptr) - ptr - end + ptr = x.ref.ptr = C.PyDict_New() + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyDict) = checknull(pyptr(x)) + +Base.iterate(x::PyDict{K,V}, it::PyRef) where {K,V} = begin + ptr = C.PyIter_Next(it) + if !isnull(ptr) + ko = C.PySequence_GetItem(ptr, 0) + isnull(ko) && pythrow() + r = C.PyObject_Convert(ko, K) + C.Py_DecRef(ko) + ism1(r) && pythrow() + k = C.takeresult(K) + vo = C.PySequence_GetItem(ptr, 1) + isnull(vo) && pythrow() + r = C.PyObject_Convert(vo, V) + C.Py_DecRef(vo) + ism1(r) && pythrow() + v = C.takeresult(V) + (k => v, it) + elseif C.PyErr_IsSet() + pythrow() else - ptr + nothing end end +Base.iterate(x::PyDict) = begin + a = C.PyObject_GetAttrString(x, "items") + isnull(a) && pythrow() + b = C.PyObject_CallNice(a) + C.Py_DecRef(a) + isnull(b) && pythrow() + it = C.PyObject_GetIter(b) + C.Py_DecRef(b) + isnull(it) && pythrow() + iterate(x, pynewref(it)) +end + +Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = pysetitem(x, convertref(K, k), convertref(V, v)) + +Base.getindex(x::PyDict{K,V}, k) where {K,V} = pygetitem(V, x, convertref(K, k)) + +Base.delete!(x::PyDict{K}, k) where {K} = pydelitem(x, convertref(K, k)) + +Base.length(x::PyDict) = Int(pylen(x)) + +Base.empty!(x::PyDict) = (@py `$x.clear()`; x) + +Base.copy(x::PyDict) = @pyv typeof(x) `$x.copy()` + +Base.haskey(x::PyDict{K}, _k) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + k === NOTIMPLEMENTED() && return false + pycontains(x, k) +end + +Base.get(x::PyDict{K}, _k, d) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + (k !== NOTIMPLEMENTED() && pycontains(x, k)) ? x[k] : d +end + +Base.get(d::Function, x::PyDict{K}, _k) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + (k !== NOTIMPLEMENTED() && pycontains(x, k)) ? x[k] : d() +end + +Base.get!(x::PyDict{K,V}, _k, d) where {K,V} = begin + k = convertref(K, _k) + pycontains(x, k) ? x[k] : (x[k] = convert(V, d)) +end + +Base.get!(d::Function, x::PyDict{K,V}, _k) where {K,V} = begin + k = convertref(K, _k) + pycontains(x, k) ? x[k] : (x[k] = convert(V, d())) +end diff --git a/src/PyException.jl b/src/PyException.jl index 5f7fc18a..fcca36a5 100644 --- a/src/PyException.jl +++ b/src/PyException.jl @@ -1,48 +1,77 @@ mutable struct PyException <: Exception - tptr :: CPyPtr - vptr :: CPyPtr - bptr :: CPyPtr - function PyException(::Val{:new}, t::Ptr=C_NULL, v::Ptr=C_NULL, b::Ptr=C_NULL, borrowed::Bool=false) - o = new(CPyPtr(t), CPyPtr(v), CPyPtr(b)) - if borrowed - C.Py_IncRef(t) - C.Py_IncRef(v) - C.Py_IncRef(b) - end - finalizer(o) do o - if CONFIG.isinitialized - tptr = getfield(o, :tptr) - vptr = getfield(o, :vptr) - bptr = getfield(o, :bptr) - if !isnull(tptr) || !isnull(vptr) || !isnull(bptr) - with_gil(false) do - C.Py_DecRef(tptr) - C.Py_DecRef(vptr) - C.Py_DecRef(bptr) - end - end - end - end - return o - end + tref :: PyRef + vref :: PyRef + bref :: PyRef + PyException(::Val{:new}, t::Ptr=C_NULL, v::Ptr=C_NULL, b::Ptr=C_NULL, borrowed::Bool=false) = + new(PyRef(Val(:new), t, borrowed), PyRef(Val(:new), v, borrowed), PyRef(Val(:new), b, borrowed)) end export PyException pythrow() = throw(PyException(Val(:new), C.PyErr_FetchTuple()...)) +""" + check(x) + +Checks if the Python error indicator is set. If so, throws the current Python exception. Otherwise returns `x`. +""" +check(x) = C.PyErr_IsSet() ? pythrow() : x + +""" + checkm1(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == -1` instead. + +If `ambig` is true the error indicator is also checked. +""" +checkm1(x::Number, ambig::Bool=false) = ism1(x) ? ambig ? check(x) : pythrow() : x + +""" + checknull(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == C_NULL` instead. + +If `ambig` is true the error indicator is also checked. +""" +checknull(x::Ptr, ambig::Bool=false) = isnull(x) ? ambig ? check(x) : pythrow() : x + +""" + checkerr(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == PYERR()` instead. + +If `ambig` is true the error indicator is also checked. +""" +checkerr(x, ambig::Bool=false) = x===PYERR() ? ambig ? check(x) : pythrow() : x + +""" + checknullconvert(T, x, ambig=false) :: T + +Same as `checknull(x, ambig)` but steals a reference to `x` and converts the result to a `T`. +""" +checknullconvert(::Type{T}, x::Ptr, ambig::Bool=false) where {T} = begin + if isnull(x) && (!ambig || C.PyErr_IsSet()) + C.Py_DecRef(x) + pythrow() + end + r = C.PyObject_Convert(x, T) + C.Py_DecRef(x) + checkm1(r) + C.takeresult(T) +end + function Base.showerror(io::IO, e::PyException) - if isnull(e.tptr) + if isnull(e.tref) print(io, "Python: mysterious error (no error was actually set)") return end if CONFIG.sysautolasttraceback err = C.Py_DecRef(C.PyImport_ImportModule("sys"), PYERR()) do sys - err = C.PyObject_SetAttrString(sys, "last_type", isnull(e.tptr) ? C.Py_None() : e.tptr) + err = C.PyObject_SetAttrString(sys, "last_type", isnull(e.tref) ? C.Py_None() : e.tref.ptr) ism1(err) && return PYERR() - err = C.PyObject_SetAttrString(sys, "last_value", isnull(e.vptr) ? C.Py_None() : e.vptr) + err = C.PyObject_SetAttrString(sys, "last_value", isnull(e.vref) ? C.Py_None() : e.vref.ptr) ism1(err) && return PYERR() - err = C.PyObject_SetAttrString(sys, "last_traceback", isnull(e.bptr) ? C.Py_None() : e.bptr) + err = C.PyObject_SetAttrString(sys, "last_traceback", isnull(e.bref) ? C.Py_None() : e.bref.ptr) ism1(err) && return PYERR() nothing end @@ -85,68 +114,53 @@ function Base.showerror(io::IO, e::PyException) print(io, "Python: ") # print the type name - tname = C.Py_DecRef(C.PyObject_GetAttrString(e.tptr, "__name__")) do tnameo - C.PyUnicode_As(tnameo, String) - end - if tname === PYERR() - C.PyErr_Clear() - print(io, "") - else + try + tname = @pyv String `$(e.tref).__name__` print(io, tname) + catch + print(io, "") end # print the error message - if !isnull(e.vptr) + if !isnull(e.vref) print(io, ": ") - vstr = C.PyObject_StrAs(e.vptr, String) - if vstr === PYERR() - C.PyErr_Clear() - print(io, "") - else + try + vstr = @pyv String `str($(e.vref))` print(io, vstr) + catch + print(io, "") end end # print the stacktrace - if !isnull(e.bptr) + if !isnull(e.bref) @label pystacktrace println(io) printstyled(io, "Python stacktrace:") - err = C.Py_DecRef(C.PyImport_ImportModule("traceback")) do tb - C.Py_DecRef(C.PyObject_GetAttrString(tb, "extract_tb")) do extr - C.Py_DecRef(C.PyObject_CallNice(extr, C.PyObjectRef(e.bptr))) do fs - nfs = C.PySequence_Length(fs) - ism1(nfs) && return PYERR() - for i in 1:nfs - println(io) - printstyled(io, " [", i, "] ") - # name - err = C.Py_DecRef(C.PySequence_GetItem(fs, i-1)) do f - name = C.Py_DecRef(C.PyObject_GetAttrString(f, "name")) do nameo - C.PyObject_StrAs(nameo, String) - end - name === PYERR() && return PYERR() - printstyled(io, name, bold=true) - printstyled(io, " at ") - fname = C.Py_DecRef(C.PyObject_GetAttrString(f, "filename")) do fnameo - C.PyObject_StrAs(fnameo, String) - end - fname === PYERR() && return PYERR() - printstyled(io, fname, ":", bold=true) - lineno = C.Py_DecRef(C.PyObject_GetAttrString(f, "lineno")) do linenoo - C.PyObject_StrAs(linenoo, String) - end - lineno === PYERR() && return PYERR() - printstyled(io, lineno, bold=true) - nothing - end - err === PYERR() && return PYERR() - end + try + @py ``` + import traceback + $(fs :: C.PyObjectRef) = fs = traceback.extract_tb($(e.bref)) + $(nfs :: Int) = len(fs) + ``` + try + for i in 1:nfs + @py ``` + f = $(fs)[$(i-1)] + $(name::String) = f.name + $(fname::String) = f.filename + $(lineno::Int) = f.lineno + ``` + println(io) + printstyled(io, " [", i, "] ") + printstyled(io, name, bold=true) + printstyled(io, " at ") + printstyled(io, fname, ":", lineno, bold=true) end + finally + C.Py_DecRef(fs) end - end - if err === PYERR() - C.PyErr_Clear() + catch print(io, "") end end diff --git a/src/PyInternedString.jl b/src/PyInternedString.jl new file mode 100644 index 00000000..f4baaec5 --- /dev/null +++ b/src/PyInternedString.jl @@ -0,0 +1,29 @@ +mutable struct PyInternedString + ref :: PyRef + val :: String + PyInternedString(x::String) = new(PyRef(), x) +end + +ispyreftype(::Type{PyInternedString}) = true +pyptr(x::PyInternedString) = begin + ptr = x.ref.ptr + if isnull(ptr) + s = Ref{CPyPtr}() + s[] = C.PyUnicode_From(x.val) + isnull(s[]) && return ptr + C.PyUnicode_InternInPlace(s) + ptr = x.ref.ptr = s[] + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyInternedString) = checknull(pyptr(x)) + +""" + pystr"..." + +An interned Python string. +""" +macro pystr_str(s::String) + PyInternedString(s) +end +export @pystr_str diff --git a/src/PyList.jl b/src/PyList.jl new file mode 100644 index 00000000..b89b6c96 --- /dev/null +++ b/src/PyList.jl @@ -0,0 +1,75 @@ +""" + PyList{T=PyObject}([o]) + +Wrap the Python list `o` (or anything satisfying the sequence interface) as a Julia vector with elements of type `T`. + +If `o` is not given, an empty list is created. +""" +struct PyList{T} <: AbstractVector{T} + ref :: PyRef + PyList{T}(o) where {T} = new{T}(PyRef(o)) + PyList{T}() where {T} = new{T}(PyRef()) +end +PyList(o) = PyList{PyObject}(o) +PyList() = PyList{PyObject}() +export PyList + +ispyreftype(::Type{<:PyList}) = true +pyptr(x::PyList) = begin + ptr = x.ref.ptr + if isnull(ptr) + ptr = x.ref.ptr = C.PyList_New(0) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyList) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyList} = C.putresult(T, T(pyborrowedref(o))) + +# Base.length(x::PyList) = @pyv Int `len($x)` +Base.length(x::PyList) = Int(checkm1(C.PyObject_Length(x))) + +Base.size(x::PyList) = (length(x),) + +Base.getindex(x::PyList{T}, i::Integer) where {T} = begin + checkbounds(x, i) + # The following line implements this function, but is typically 3x slower. + # @pyv T `$x[$(i-1)]` + p = checknull(C.PySequence_GetItem(x, i-1)) + r = C.PyObject_Convert(p, T) + C.Py_DecRef(p) + checkm1(r) + C.takeresult(T) +end + +Base.setindex!(x::PyList{T}, _v, i::Integer) where {T} = begin + checkbounds(x, i) + v = convertref(T, _v) + # The following line implements this function, but is typically 10x slower + # @py `$x[$(i-1)] = $v` + vp = checknull(C.PyObject_From(v)) + err = C.PySequence_SetItem(x, i-1, vp) + C.Py_DecRef(vp) + checkm1(err) + x +end + +Base.insert!(x::PyList{T}, i::Integer, v) where {T} = (i==length(x)+1 || checkbounds(x, i); @py `$x.insert($(i-1), $(convertref(T, v)))`; x) + +Base.push!(x::PyList{T}, v) where {T} = (@py `$x.append($(convertref(T, v)))`; x) + +Base.pushfirst!(x::PyList, v) = insert!(x, 1, v) + +Base.pop!(x::PyList{T}) where {T} = @pyv T `$x.pop()` + +Base.popat!(x::PyList{T}, i::Integer) where {T} = (checkbounds(x, i); @pyv T `$x.pop($(i-1))`) + +Base.popfirst!(x::PyList) = pop!(x, 1) + +Base.reverse!(x::PyList) = (@py `$x.reverse()`; x) + +# TODO: support kwarg `by` (becomes python kwarg `key`) +Base.sort!(x::PyList; rev::Bool=false) = (@py `$x.sort(reverse=$rev)`; x) + +Base.empty!(x::PyList) = (@py `$x.clear()`; x) + +Base.copy(x::PyList) = @pyv typeof(x) `$x.copy()` diff --git a/src/PyObject.jl b/src/PyObject.jl new file mode 100644 index 00000000..16dad24c --- /dev/null +++ b/src/PyObject.jl @@ -0,0 +1,132 @@ +mutable struct PyObject + ref :: PyRef + make :: Any + PyObject(::Val{:nocopy}, o::PyRef) = new(o) + PyObject(o) = new(PyRef(o)) + PyObject(::Val{:lazy}, mk) = new(PyRef(), mk) +end +export PyObject + +ispyreftype(::Type{PyObject}) = true +pyptr(o::PyObject) = begin + ref = getfield(o, :ref) + ptr = ref.ptr + if isnull(ptr) + val = try + getfield(o, :make)() + catch err + C.PyErr_SetString(C.PyExc_Exception(), "Error retrieving object value: $err") + return ptr + end + ptr = ref.ptr = C.PyObject_From(val) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, o::PyObject) = checknull(pyptr(o)) +pynewobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyObject(Val(:nocopy), pynewref(p)) +pyborrowedobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyObject(Val(:nocopy), pyborrowedref(p)) +pylazyobject(mk) = PyObject(Val(:lazy), mk) + +C.PyObject_TryConvert__initial(o, ::Type{PyObject}) = C.putresult(PyObject, pyborrowedobject(o)) + +Base.convert(::Type{PyObject}, x::PyObject) = x +Base.convert(::Type{PyObject}, x) = PyObject(x) + +### Cache some common values + +const _pynone = pylazyobject(() -> pynone(PyRef)) +pynone(::Type{PyObject}) = _pynone + +const _pytrue = pylazyobject(() -> pybool(PyRef, true)) +const _pyfalse = pylazyobject(() -> pybool(PyRef, false)) +pybool(::Type{PyObject}, x::Bool) = x ? _pytrue : _pyfalse + +### IO + +Base.string(o::PyObject) = pystr(String, o) + +Base.print(io::IO, o::PyObject) = print(io, string(o)) + +Base.show(io::IO, o::PyObject) = begin + s = pyrepr(String, o) + if get(io, :typeinfo, Any) == typeof(o) + print(io, s) + elseif startswith(s, "<") && endswith(s, ">") + print(io, "") + end +end + +### PROPERTIES + +Base.getproperty(o::PyObject, k::Symbol) = pygetattr(PyObject, o, k) + +Base.setproperty!(o::PyObject, k::Symbol, v) = pysetattr(o, k, v) + +Base.hasproperty(o::PyObject, k::Symbol) = pyhasattr(o, k) + +function Base.propertynames(o::PyObject) + # this follows the logic of rlcompleter.py + @py ``` + def members(o): + def classmembers(c): + r = dir(c) + if hasattr(c, '__bases__'): + for b in c.__bases__: + r += classmembers(b) + return r + words = set(dir(o)) + words.discard('__builtins__') + if hasattr(o, '__class__'): + words.add('__class__') + words.update(classmembers(o.__class__)) + return words + $words = members($o) + ``` + [Symbol(pystr(String, x)) for x in words] +end + +### CALL + +(f::PyObject)(args...; kwargs...) = pycall(PyObject, f, args...; kwargs...) + +### ITERATE & INDEX + +Base.IteratorSize(::Type{PyObject}) = Base.SizeUnknown() +Base.eltype(::Type{PyObject}) = PyObject + +Base.getindex(o::PyObject, k) = pygetitem(PyObject, o, k) +Base.getindex(o::PyObject, k...) = pygetitem(PyObject, o, k) +Base.setindex!(o::PyObject, v, k) = pysetitem(o, k, v) +Base.setindex!(o::PyObject, v, k...) = pysetitem(o, k, v) +Base.delete!(o::PyObject, k) = pydelitem(o, k) +Base.delete!(o::PyObject, k...) = pydelitem(o, k) + +Base.length(o::PyObject) = Int(pylen(o)) + +Base.iterate(o::PyObject, it::PyRef) = begin + vo = C.PyIter_Next(it) + if !isnull(vo) + (pynewobject(vo), it) + elseif C.PyErr_IsSet() + pythrow() + else + nothing + end +end +Base.iterate(o::PyObject) = iterate(o, pynewref(C.PyObject_GetIter(o), true)) + +Base.in(x, o::PyObject) = pycontains(o, x) +Base.hash(o::PyObject) = trunc(UInt, pyhash(o)) + +### COMPARISON + +Base.:(==)(x::PyObject, y::PyObject) = pyeq(PyObject, x, y) +Base.:(!=)(x::PyObject, y::PyObject) = pynq(PyObject, x, y) +Base.:(<=)(x::PyObject, y::PyObject) = pyle(PyObject, x, y) +Base.:(< )(x::PyObject, y::PyObject) = pylt(PyObject, x, y) +Base.:(>=)(x::PyObject, y::PyObject) = pyge(PyObject, x, y) +Base.:(> )(x::PyObject, y::PyObject) = pygt(PyObject, x, y) +Base.isequal(x::PyObject, y::PyObject) = pyeq(Bool, x, y) +Base.isless(x::PyObject, y::PyObject) = pylt(Bool, x, y) diff --git a/src/PyRef.jl b/src/PyRef.jl new file mode 100644 index 00000000..8ad6bc2e --- /dev/null +++ b/src/PyRef.jl @@ -0,0 +1,49 @@ +""" + PyRef([x]) + +A reference to a Python object converted from `x`, or a null reference if `x` is not given. + +This is baically just a mutable reference to a pointer to a Python object. +It owns the reference (if non-NULL) and automatically decrefs it when finalized. + +Building block for more complex wrapper types such as `PyObject` and `PyList`. +""" +mutable struct PyRef + ptr :: CPyPtr + PyRef(::Val{:new}, ptr::Ptr, borrowed::Bool) = begin + x = new(CPyPtr(ptr)) + borrowed && C.Py_IncRef(ptr) + finalizer(x) do x + if !CONFIG.isinitialized + ptr = x.ptr + if !isnull(ptr) + with_gil(false) do + C.Py_DecRef(ptr) + end + x.ptr = CPyPtr() + end + end + end + x + end +end +export PyRef + +pynewref(x::Ptr, check::Bool=false) = (check && isnull(x)) ? pythrow() : PyRef(Val(:new), x, false) +pyborrowedref(x::Ptr, check::Bool=false) = (check && isnull(x)) ? pythrow() : PyRef(Val(:new), x, true) + +ispyreftype(::Type{PyRef}) = true +pyptr(x::PyRef) = x.ptr +isnull(x::PyRef) = isnull(pyptr(x)) +Base.unsafe_convert(::Type{CPyPtr}, x::PyRef) = pyptr(x) +C.PyObject_TryConvert__initial(o, ::Type{PyRef}) = C.putresult(PyRef, pyborrowedref(o)) + +PyRef(x) = begin + ptr = C.PyObject_From(x) + isnull(ptr) && pythrow() + pynewref(ptr) +end +PyRef() = pynewref(CPyPtr()) + +Base.convert(::Type{PyRef}, x::PyRef) = x +Base.convert(::Type{PyRef}, x) = PyRef(x) diff --git a/src/PySet.jl b/src/PySet.jl new file mode 100644 index 00000000..8b2c416f --- /dev/null +++ b/src/PySet.jl @@ -0,0 +1,83 @@ +""" + PySet{T=PyObject}([o]) + +Wrap the Python set `o` (or anything satisfying the set interface) as a Julia set with elements of type `T`. + +If `o` is not given, an empty set is created. +""" +struct PySet{T} <: AbstractSet{T} + ref :: PyRef + PySet{T}(o) where {T} = new{T}(PyRef(o)) + PySet{T}() where {T} = new{T}(PyRef()) +end +PySet(o) = PySet{PyObject}(o) +PySet() = PySet{PyObject}() +export PySet + +ispyreftype(::Type{<:PySet}) = true +pyptr(x::PySet) = begin + ptr = x.ref.ptr + if isnull(ptr) + ptr = x.ref.ptr = C.PySet_New(C_NULL) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PySet) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PySet} = C.putresult(T, T(pyborrowedref(o))) + +Base.iterate(x::PySet{T}, it::PyRef) where {T} = begin + ptr = C.PyIter_Next(it) + if ptr != C_NULL + r = C.PyObject_Convert(ptr, T) + C.Py_DecRef(ptr) + checkm1(r) + (C.takeresult(T), it) + elseif C.PyErr_IsSet() + pythrow() + else + nothing + end +end +Base.iterate(x::PySet) = begin + it = C.PyObject_GetIter(x) + isnull(it) && pythrow() + iterate(x, pynewref(it)) +end + +# Base.length(x::PySet) = @pyv Int `len($x)` +Base.length(x::PySet) = Int(checkm1(C.PyObject_Length(x))) + +Base.in(_v, x::PySet{T}) where {T} = begin + v = tryconvertref(T, _v) + v === PYERR() && pythrow() + v === NOTIMPLEMENTED() && return false + pycontains(x, v) +end + +Base.push!(x::PySet{T}, v) where {T} = (@py `$x.add($(convertref(T, v)))`; x) + +Base.delete!(x::PySet{T}, _v) where {T} = begin + v = tryconvertref(T, _v) + v === PYERR() && pythrow() + v === NOTIMPLEMENTED() && return x + @py `$x.discard($v)` + x +end + +Base.pop!(x::PySet{T}) where {T} = @pyv T `$x.pop()` + +Base.pop!(x::PySet{T}, _v) where {T} = begin + v = tryconvert(T, _v) + v === PYERR() && pythrow() + (v !== NOTIMPLEMENTED() && v in x) ? (delete!(x, v); v) : error("not an element") +end + +Base.pop!(x::PySet{T}, _v, d) where {T} = begin + v = tryconvert(T, _v) + v === PYERR() && pythrow() + (v !== NOTIMPLEMENTED() && v in x) ? (delete!(x, v); v) : d +end + +Base.empty!(x::PySet) = (@py `$x.clear()`; x) + +Base.copy(x::PySet) = @pyv typeof(x) `$x.copy()` diff --git a/src/Python.jl b/src/Python.jl index a23d2d24..c3d7d0af 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -52,11 +52,26 @@ Base.show(io::IO, ::MIME"text/plain", c::Config) = end const CONFIG = Config() -# Used to signal a Python error from functions that return general Julia objects -struct PYERR end +""" + ispyreftype(::Type{T}) -# Used to signal that conversion failed -struct NOTIMPLEMENTED end +True if `T` is a wrapper type for a single Python reference. + +Such objects must implement: +- `pyptr(o::T) =` the underlying pointer (or NULL on error) +- `unsafe_convert(::Type{CPyPtr}, o::T) = checknull(pyptr(o))` +""" +ispyreftype(::Type) = false + +""" + pyptr(o) + +Retrieve the underlying Python object pointer from o. +""" +function pyptr end + +convertref(::Type{T}, x) where {T} = ispyreftype(T) ? x : convert(T, x) +tryconvertref(::Type{T}, x) where {T} = ispyreftype(T) ? x : tryconvert(T, x) # C API include("cpython/CPython.jl") @@ -72,12 +87,17 @@ const CPyPtr = C.PyPtr # include("error.jl") # include("import.jl") include("gil.jl") +include("eval.jl") +include("builtins.jl") -include("PyException.jl") +include("PyRef.jl") include("PyCode.jl") +include("PyInternedString.jl") +include("PyException.jl") +include("PyObject.jl") include("PyDict.jl") - -include("eval.jl") +include("PyList.jl") +include("PySet.jl") # # abstract interfaces # include("number.jl") diff --git a/src/builtins.jl b/src/builtins.jl new file mode 100644 index 00000000..1716f4cf --- /dev/null +++ b/src/builtins.jl @@ -0,0 +1,628 @@ +cpyop(f::Function, x) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + r = f(xo) + C.Py_DecRef(xo) + r +end + +cpyop(f::Function, x, y) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + yo = C.PyObject_From(y) + isnull(yo) && (C.Py_DecRef(xo); pythrow()) + r = f(xo, yo) + C.Py_DecRef(xo) + C.Py_DecRef(yo) + r +end + +cpyop(f::Function, x, y, z) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + yo = C.PyObject_From(y) + isnull(yo) && (C.Py_DecRef(xo); pythrow()) + zo = C.PyObject_From(z) + isnull(yo) && (C.Py_DecRef(xo); C.Py_DecRef(yo); pythrow()) + r = f(xo, yo, zo) + C.Py_DecRef(xo) + C.Py_DecRef(yo) + C.Py_DecRef(zo) + r +end + +cpyop(::Type{T}, f::Function, args...) where {T} = checknullconvert(T, cpyop(f, args...)) + +""" + pyconvert(T, x) :: T + +Convert Python object `x` to a `T`. +""" +pyconvert(::Type{T}, x) where {T} = checknullconvert(T, C.PyObject_From(x)) + +""" + pyis(x, y) :: Bool + +Equivalent to `x is y` in Python. +""" +pyis(x::X, y::Y) where {X,Y} = begin + if ispyreftype(X) + xo = pyptr(x) + isnull(xo) && pythrow() + else + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + C.Py_DecRef(xo) + end + if ispyreftype(Y) + yo = pyptr(y) + isnull(yo) && pythrow() + else + yo = C.PyObject_From(y) + isnull(yo) && pythrow() + C.Py_DecRef(yo) + end + xo == yo +end + +""" + pyhasattr(x, k) :: Bool + +Equivalent to `hasattr(x, k)` in Python, returned as a `Bool`. +""" +pyhasattr(x, k) = checkm1(cpyop(C.PyObject_HasAttr, x, k)) != 0 +pyhasattr(x, k::String) = checkm1(cpyop(xo -> C.PyObject_HasAttrString(xo, k), x)) != 0 +pyhasattr(x, k::Symbol) = pyhasattr(x, string(k)) +export pyhasattr + +""" + pygetattr([T=PyObject,] x, k) :: T + +Equivalent to `x.k` or `getattr(x, k)` in Python. +""" +pygetattr(::Type{T}, x, k) where {T} = cpyop(T, C.PyObject_GetAttr, x, k) +pygetattr(::Type{T}, x, k::String) where {T} = cpyop(T, xo->C.PyObject_GetAttrString(xo, k), x) +pygetattr(::Type{T}, x, k::Symbol) where {T} = pygetattr(T, x, string(k)) +pygetattr(x, k) = pygetattr(PyObject, x, k) +export pygetattr + +""" + pysetattr(x, k, v) + +Equivalent to `x.k = v` or `setattr(x, k, v)` in Python, but returns `x`. +""" +pysetattr(x, k, v) = (checkm1(cpyop(C.PyObject_SetAttr, x, k, v)); x) +pysetattr(x, k::String, v) = (checkm1(cpyop((xo, vo) -> C.PyObject_SetAttrString(xo, k, vo), x, v)); x) +pysetattr(x, k::Symbol, v) = pysetattr(x, string(k), v) +export pysetattr + +""" + pydir([T=PyObject,] x) :: T + +Equivalent to `dir(x)` in Python. +""" +pydir(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Dir, x) +pydir(x) = pydir(PyObject, x) +export pydir + +""" + pycall([T=PyObject,] f, args...; kwargs...) :: T + +Equivalent to `f(*args, **kwargs)` in Python. +""" +pycall(::Type{T}, f, args...; opts...) where {T} = cpyop(T, fo -> C.PyObject_CallArgs(fo, args, opts), f) +pycall(f, args...; opts...) = pycall(PyObject, f, args...; opts...) +export pycall + +""" + pyrepr([T=PyObject,] x) :: T + +Equivalent to `repr(x)` in Python. +""" +pyrepr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Repr, x) +pyrepr(x) = pyrepr(PyObject, x) +export pyrepr + +""" + pystr([T=PyObject,] x) :: T + +Equivalent to `str(x)` in Python. +""" +pystr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Repr, x) +pystr(x) = pystr(PyObject, x) +export pystr + +""" + pylen(x) :: Integer + +Equivalent to `len(x)` in Python. +""" +pylen(x) = checkm1(cpyop(C.PyObject_Length, x)) + +""" + pycontains(x, v) :: Bool + +Equivalent to `v in x` in Python. +""" +pycontains(x, v) = checkm1(cpyop(C.PySequence_Contains, x, v)) != 0 +export pycontains + +""" + pyin(v, x) :: Bool + +Equivalent to `v in x` in Python. +""" +pyin(v, x) = pycontains(x, v) +export pyin + +""" + pygetitem([T=PyObject,] x, k) :: T + +Equivalent to `x[k]` or `getitem(x, k)` in Python. +""" +pygetitem(::Type{T}, x, k) where {T} = cpyop(T, C.PyObject_GetItem, x, k) +pygetitem(x, k) = pygetitem(PyObject, x, k) +export pygetitem + +""" + pysetitem(x, k, v) + +Equivalent to `x[k] = v` or `setitem(x, k, v)` in Python, but returns `x`. +""" +pysetitem(x, k, v) = (checkm1(cpyop(C.PyObject_SetItem, x, k, v)); x) +export pysetitem + +""" + pydelitem(x, k) + +Equivalent to `del x[k]` or `delitem(x, k)` in Python, but returns x. +""" +pydelitem(x, k) = (checkm1(cpyop(C.PyObject_DelItem, x, k)); x) +export pydelitem + +""" + pynone([T=PyObject]) :: T + +Equivalent to `None` in Python. +""" +pynone(::Type{T}) where {T} = (checkm1(C.PyObject_Convert(C.Py_None(), T)); takeresult(T)) +pynone() = pynone(PyObject) +export pynone + +""" + pybool([T=PyObject,] ...) :: T + +Equivalent to `bool(...)` in Python. +""" +pybool(::Type{T}, x::Bool) where {T} = checknullconvert(T, C.PyBool_From(x)) +pybool(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyBool_Type(), args, kwargs)) +pybool(args...; kwargs...) = pybool(PyObject, args...; kwargs...) +export pybool + +""" + pyint([T=PyObject,] ...) :: T + +Equivalent to `int(...)` in Python. +""" +pyint(::Type{T}, x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) where {T} = + checknullconvert(T, C.PyLong_From(x)) +pyint(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Long, x) +pyint(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyLong_Type(), args, kwargs)) +pyint(args...; kwargs...) = pyint(PyObject, args...; kwargs...) +export pyint + +""" + pyfloat([T=PyObject,] ...) :: T + +Equivalent to `float(...)` in Python. +""" +pyfloat(::Type{T}, x::Union{Float16,Float32,Float64}) where {T} = checknullconvert(T, C.PyFloat_From(x)) +pyfloat(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Float, x) +pyfloat(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyFloat_Type(), args, kwargs)) +pyfloat(args...; kwargs...) = pyfloat(PyObject, args...; kwargs...) +export pyfloat + +""" + pyimport([T=PyObject,] name) :: T + pyimport([T=PyObject,] name=>attr) :: T + pyimport([T=PyObject,] name=>(attr,...)) :: Tuple{T,...} + +Imports and returns the Python module `name`. + +If additionally `attr` is given, the given attribute of the module is returned instead. +It may also be a tuple of attributes. + +If several arguments are given, each one is imported and a tuple is returned. +""" +pyimport(::Type{T}, m) where {T} = cpyop(T, C.PyImport_Import, m) +pyimport(::Type{T}, m::String) where {T} = checknullconvert(T, C.PyImport_ImportModule(m)) +pyimport(::Type{T}, m::Pair) where {T} = begin + r = pyimport(PyObject, m[1]) + m[2] isa Tuple ? map(k->pygetattr(T, r, k), m[2]) : pygetattr(T, r, m[2]) +end +pyimport(::Type{T}, m1, m2, ms...) where {T} = map(m->pyimport(T, m), (m1, m2, ms...)) +pyimport(m1, ms...) = pyimport(PyObject, m1, ms...) +export pyimport + +""" + pytruth(x) :: Bool + +The truthyness of `x`, equivalent to `bool(x)` or `not not x` in Python, or to `pybool(Bool, x)`. +""" +pytruth(x) = checkm1(cpyop(C.PyObject_IsTrue, x)) != 0 +export pytruth + +""" + pyissubclass(x, y) :: Bool + +Equivalent to `issubclass(x, y)` in Python. +""" +pyissubclass(x, y) = checkm1(cpyop(C.PyObject_IsSubclass, x, y)) != 0 +export pyissubclass + +""" + pyisinstance(x, y) :: Bool + +Equivalent to `isinstance(x, y)` in Python. +""" +pyisinstance(x, y) = checkm1(cpyop(C.PyObject_IsInstance, x, y)) != 0 +export pyisinstance + +""" + pyhash(x) :: Integer + +Equivalent to `hash(x)` in Python. +""" +pyhash(x) = checkm1(cpyop(C.PyObject_Hash, x)) +export pyhash + +""" + pycompare([T=PyObject,] x, op, y) :: T + +Equivalent to `x op y` in Python where `op` is one of: `=`, `≠`, `<`, `≤`, `>`, `≥`. +""" +pycompare(::Type{T}, x, ::typeof(==), y) where {T} = _pycompare(T, x, C.Py_EQ, y) +pycompare(::Type{T}, x, ::typeof(!=), y) where {T} = _pycompare(T, x, C.Py_NE, y) +pycompare(::Type{T}, x, ::typeof(< ), y) where {T} = _pycompare(T, x, C.Py_LT, y) +pycompare(::Type{T}, x, ::typeof(<=), y) where {T} = _pycompare(T, x, C.Py_LE, y) +pycompare(::Type{T}, x, ::typeof(> ), y) where {T} = _pycompare(T, x, C.Py_GT, y) +pycompare(::Type{T}, x, ::typeof(>=), y) where {T} = _pycompare(T, x, C.Py_GE, y) +pycompare(x, op, y) = pycompare(PyObject, x, op, y) +_pycompare(::Type{T}, x, op::Cint, y) where {T} = cpyop(T, (xo,yo)->C.PyObject_RichCompare(xo, yo, op), x, y) +_pycompare(::Type{Bool}, x, op::Cint, y) = checkm1(cpyop((xo,yo)->C.PyObject_RichCompareBool(xo, yo, op), x, y)) != 0 +export pycompare + +""" + pyeq([T=PyObject,] x, y) :: T + +Equivalent to `x == y` in Python. +""" +pyeq(::Type{T}, x, y) where {T} = pycompare(T, x, ==, y) +pyeq(x, y) = pycompare(x, ==, y) +export pyeq + +""" + pyne([T=PyObject,] x, y) :: T + +Equivalent to `x != y` in Python. +""" +pyne(::Type{T}, x, y) where {T} = pycompare(T, x, !=, y) +pyne(x, y) = pycompare(x, !=, y) +export pyne + +""" + pyge([T=PyObject,] x, y) :: T + +Equivalent to `x >= y` in Python. +""" +pyge(::Type{T}, x, y) where {T} = pycompare(T, x, >=, y) +pyge(x, y) = pycompare(x, >=, y) +export pyge + +""" + pygt([T=PyObject,] x, y) :: T + +Equivalent to `x > y` in Python. +""" +pygt(::Type{T}, x, y) where {T} = pycompare(T, x, >, y) +pygt(x, y) = pycompare(x, >, y) +export pygt + +""" + pyle([T=PyObject,] x, y) :: T + +Equivalent to `x <= y` in Python. +""" +pyle(::Type{T}, x, y) where {T} = pycompare(T, x, <=, y) +pyle(x, y) = pycompare(x, <=, y) +export pyle + +""" + pylt([T=PyObject,] x, y) :: T + +Equivalent to `x < y` in Python. +""" +pylt(::Type{T}, x, y) where {T} = pycompare(T, x, <, y) +pylt(x, y) = pycompare(x, <, y) +export pylt + +""" + pyadd([T=PyObject,] x, y) :: T + +Equivalent to `x + y` in Python. +""" +pyadd(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Add, x, y) +pyadd(x, y) = pyadd(PyObject, x, y) +export pyadd + +""" + pyiadd([T=typeof(x),] x, y) :: T + +`x = pyiadd(x, y)` is equivalent to `x += y` in Python. +""" +pyiadd(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceAdd, x, y) +pyiadd(x, y) = pyiadd(typeof(x), x, y) +export pyiadd + +""" + pysub([T=PyObject,] x, y) :: T + +Equivalent to `x - y` in Python. +""" +pysub(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Subtract, x, y) +pysub(x, y) = pysub(PyObject, x, y) +export pysub + +""" + pyisub([T=typeof(x),] x, y) :: T + +`x = pyisub(x, y)` is equivalent to `x -= y` in Python. +""" +pyisub(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceSubtract, x, y) +pyisub(x, y) = pyisub(typeof(x), x, y) +export pyisub + +""" + pymul([T=PyObject,] x, y) :: T + +Equivalent to `x * y` in Python. +""" +pymul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Multiply, x, y) +pymul(x, y) = pymul(PyObject, x, y) +export pymul + +""" + pyimul([T=typeof(x),] x, y) :: T + +`x = pyimul(x, y)` is equivalent to `x *= y` in Python. +""" +pyimul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceMultiply, x, y) +pyimul(x, y) = pyimul(typeof(x), x, y) +export pyimul + +""" + pymatmul([T=PyObject,] x, y) :: T + +Equivalent to `x @ y` in Python. +""" +pymatmul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_MatrixMultiply, x, y) +pymatmul(x, y) = pymatmul(PyObject, x, y) +export pymatmul + +""" + pyimatmul([T=typeof(x),] x, y) :: T + +`x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. +""" +pyimatmul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceMatrixMultiply, x, y) +pyimatmul(x, y) = pyimatmul(typeof(x), x, y) +export pyimatmul + +""" + pyfloordiv([T=PyObject,] x, y) :: T + +Equivalent to `x // y` in Python. +""" +pyfloordiv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_FloorDivide, x, y) +pyfloordiv(x, y) = pyfloordiv(PyObject, x, y) +export pyfloordiv + +""" + pyifloordiv([T=typeof(x),] x, y) :: T + +`x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. +""" +pyifloordiv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceFloorDivide, x, y) +pyifloordiv(x, y) = pyifloordiv(typeof(x), x, y) +export pyifloordiv + +""" + pytruediv([T=PyObject,] x, y) :: T + +Equivalent to `x / y` in Python. +""" +pytruediv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_TrueDivide, x, y) +pytruediv(x, y) = pytruediv(PyObject, x, y) +export pytruediv + +""" + pyitruediv([T=typeof(x),] x, y) :: T + +`x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. +""" +pyitruediv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceTrueDivide, x, y) +pyitruediv(x, y) = pyitruediv(typeof(x), x, y) +export pyitruediv + +""" + pyrem([T=PyObject,] x, y) :: T + +Equivalent to `x % y` in Python. +""" +pyrem(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Remainder, x, y) +pyrem(x, y) = pyrem(PyObject, x, y) +export pyrem + +""" + pyirem([T=typeof(x),] x, y) :: T + +`x = pyirem(x, y)` is equivalent to `x %= y` in Python. +""" +pyirem(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceRemainder, x, y) +pyirem(x, y) = pyirem(typeof(x), x, y) +export pyirem + +""" + pydivmod([T=PyObject,] x, y) :: T + +Equivalent to `divmod(x, y)` in Python. +""" +pydivmod(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_DivMod, x, y) +pydivmod(x, y) = pydivmod(PyObject, x, y) +export pydivmod + +""" + pylshift([T=PyObject,] x, y) :: T + +Equivalent to `x << y` in Python. +""" +pylshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Lshift, x, y) +pylshift(x, y) = pylshift(PyObject, x, y) +export pylshift + +""" + pyilshift([T=typeof(x),] x, y) :: T + +`x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. +""" +pyilshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceLshift, x, y) +pyilshift(x, y) = pyilshift(typeof(x), x, y) +export pyilshift + +""" + pyrshift([T=PyObject,] x, y) :: T + +Equivalent to `x >> y` in Python. +""" +pyrshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Rshift, x, y) +pyrshift(x, y) = pyrshift(PyObject, x, y) +export pyrshift + +""" + pyirshift([T=typeof(x),] x, y) :: T + +`x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. +""" +pyirshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceRshift, x, y) +pyirshift(x, y) = pyirshift(typeof(x), x, y) +export pyirshift + +""" + pyand([T=PyObject,] x, y) :: T + +Equivalent to `x & y` in Python. +""" +pyand(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_And, x, y) +pyand(x, y) = pyand(PyObject, x, y) +export pyand + +""" + pyiand([T=typeof(x),] x, y) :: T + +`x = pyiand(x, y)` is equivalent to `x &= y` in Python. +""" +pyiand(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceAnd, x, y) +pyiand(x, y) = pyiand(typeof(x), x, y) +export pyiand + +""" + pyxor([T=PyObject,] x, y) :: T + +Equivalent to `x ^ y` in Python. +""" +pyxor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Xor, x, y) +pyxor(x, y) = pyxor(PyObject, x, y) +export pyxor + +""" + pyixor([T=typeof(x),] x, y) :: T + +`x = pyixor(x, y)` is equivalent to `x ^= y` in Python. +""" +pyixor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceXor, x, y) +pyixor(x, y) = pyixor(typeof(x), x, y) +export pyixor + +""" + pyor([T=PyObject,] x, y) :: T + +Equivalent to `x | y` in Python. +""" +pyor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Or, x, y) +pyor(x, y) = pyor(PyObject, x, y) +export pyor + +""" + pyior([T=typeof(x),] x, y) :: T + +`x = pyior(x, y)` is equivalent to `x |= y` in Python. +""" +pyior(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceOr, x, y) +pyior(x, y) = pyior(typeof(x), x, y) +export pyior + +""" + pypow([T=PyObject,] x, y, [z]) :: T + +Equivalent to `x**y` or `pow(x, y, z)` in Python. +""" +pypow(::Type{T}, x, y, z=C.PyObjectRef(C.Py_None())) where {T} = cpyop(T, C.PyNumber_Power, x, y, z) +pypow(x, y, z) = pypow(PyObject, x, y, z) +pypow(x, y) = pypow(PyObject, x, y) +export pypow + +""" + pyipow([T=typeof(x),] x, y, [z]) :: T + +`x = pyipow(x, y)` is equivalent to `x **= y` in Python. +""" +pyipow(::Type{T}, x, y, z=C.PyObjectRef(C.Py_None())) where {T} = cpyop(T, C.PyNumber_InPlacePower, x, y, z) +pyipow(x, y, z) = pyipow(typeof(x), x, y, z) +pyipow(x, y) = pyipow(typeof(x), x, y) +export pyipow + +""" + pyneg([T=typeof(x),] x) :: T + +Equivalent to `-x` in Python. +""" +pyneg(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Negative, x) +pyneg(x) = pyneg(typeof(x), x) +export pyneg + +""" + pypos([T=typeof(x),] x) :: T + +Equivalent to `+x` in Python. +""" +pypos(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Positive, x) +pypos(x) = pypos(typeof(x), x) +export pypos + +""" + pyabs([T=typeof(x),] x) :: T + +Equivalent to `abs(x)` in Python. +""" +pyabs(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Absolute, x) +pyabs(x) = pyabs(typeof(x), x) +export pyabs + +""" + pyinv([T=typeof(x),] x) :: T + +Equivalent to `-x` in Python. +""" +pyinv(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Invert, x) +pyinv(x) = pyinv(typeof(x), x) +export pyinv diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index bf93fcd4..49f08c63 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -1,331 +1,12 @@ module CPython using Libdl -using ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect +import ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect, tryconvert, ispyreftype, pyptr, putresult, takeresult, moveresult, CACHE, Python using Base: @kwdef using UnsafePointers: UnsafePtr -tryconvert(::Type{T}, x::PYERR) where {T} = PYERR() -tryconvert(::Type{T}, x::NOTIMPLEMENTED) where {T} = NOTIMPLEMENTED() -tryconvert(::Type{T}, x::T) where {T} = x -tryconvert(::Type{T}, x) where {T} = - try - convert(T, x) - catch - NOTIMPLEMENTED() - end - -@enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 - -const Py_single_input = 256 -const Py_file_input = 257 -const Py_eval_input = 258 -const Py_func_type_input = 345 - -const Py_LT = Cint(0) -const Py_LE = Cint(1) -const Py_EQ = Cint(2) -const Py_NE = Cint(3) -const Py_GT = Cint(4) -const Py_GE = Cint(5) - -const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments -const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs -const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) -const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) -const Py_METH_CLASS = 0x0010 # for class methods -const Py_METH_STATIC = 0x0020 # for static methods - -const Py_T_SHORT =0 -const Py_T_INT =1 -const Py_T_LONG =2 -const Py_T_FLOAT =3 -const Py_T_DOUBLE =4 -const Py_T_STRING =5 -const Py_T_OBJECT =6 -const Py_T_CHAR =7 -const Py_T_BYTE =8 -const Py_T_UBYTE =9 -const Py_T_USHORT =10 -const Py_T_UINT =11 -const Py_T_ULONG =12 -const Py_T_STRING_INPLACE =13 -const Py_T_BOOL =14 -const Py_T_OBJECT_EX =16 -const Py_T_LONGLONG =17 # added in Python 2.5 -const Py_T_ULONGLONG =18 # added in Python 2.5 -const Py_T_PYSSIZET =19 # added in Python 2.6 -const Py_T_NONE =20 # added in Python 3.0 - -const Py_READONLY = 1 -const Py_READ_RESTRICTED = 2 -const Py_WRITE_RESTRICTED = 4 -const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) - -const PyBUF_MAX_NDIM = 64 - -# Flags for getting buffers -const PyBUF_SIMPLE = 0x0 -const PyBUF_WRITABLE = 0x0001 -const PyBUF_WRITEABLE = PyBUF_WRITABLE -const PyBUF_FORMAT = 0x0004 -const PyBUF_ND = 0x0008 -const PyBUF_STRIDES = (0x0010 | PyBUF_ND) -const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) -const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) -const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) -const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) - -const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) -const PyBUF_CONTIG_RO = (PyBUF_ND) - -const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) -const PyBUF_STRIDED_RO = (PyBUF_STRIDES) - -const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) -const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) - -const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) -const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) - -const PyBUF_READ = 0x100 -const PyBUF_WRITE = 0x200 - -# Python 2.7 -const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) -const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) -const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 -const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) -const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) -const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) -const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) -const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) -const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) -const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) -const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) -const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) - -# Python 3.0+ has only these: -const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) -const Py_TPFLAGS_BASETYPE = (0x00000001<<10) -const Py_TPFLAGS_READY = (0x00000001<<12) -const Py_TPFLAGS_READYING = (0x00000001<<13) -const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) -const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) -const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) -const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) -const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) -const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) -const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) -const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) -const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) -const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) -const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) -const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) -const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) - -# only use this if we have the stackless extension -const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) - -const Py_hash_t = Cssize_t -const Py_ssize_t = Cssize_t - -@kwdef struct Py_complex - real :: Cdouble = 0.0 - imag :: Cdouble = 0.0 -end - -@kwdef struct PyObject - # assumes _PyObject_HEAD_EXTRA is empty - refcnt :: Py_ssize_t = 0 - type :: Ptr{PyObject} = C_NULL -end - -const PyPtr = Ptr{PyObject} - -struct PyObjectRef - ptr :: PyPtr -end -Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr - -@kwdef struct PyVarObject - ob_base :: PyObject = PyObject() - size :: Py_ssize_t = 0 -end - -@kwdef struct PyMethodDef - name :: Cstring = C_NULL - meth :: Ptr{Cvoid} = C_NULL - flags :: Cint = 0 - doc :: Cstring = C_NULL -end - -@kwdef struct PyGetSetDef - name :: Cstring = C_NULL - get :: Ptr{Cvoid} = C_NULL - set :: Ptr{Cvoid} = C_NULL - doc :: Cstring = C_NULL - closure :: Ptr{Cvoid} = C_NULL -end - -@kwdef struct PyMemberDef - name :: Cstring = C_NULL - typ :: Cint = 0 - offset :: Py_ssize_t = 0 - flags :: Cint = 0 - doc :: Cstring = C_NULL -end - -@kwdef struct PyNumberMethods - add :: Ptr{Cvoid} = C_NULL # (o,o)->o - subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o - power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - negative :: Ptr{Cvoid} = C_NULL # (o)->o - positive :: Ptr{Cvoid} = C_NULL # (o)->o - absolute :: Ptr{Cvoid} = C_NULL # (o)->o - bool :: Ptr{Cvoid} = C_NULL # (o)->Cint - invert :: Ptr{Cvoid} = C_NULL # (o)->o - lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - and :: Ptr{Cvoid} = C_NULL # (o,o)->o - xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - or :: Ptr{Cvoid} = C_NULL # (o,o)->o - int :: Ptr{Cvoid} = C_NULL # (o)->o - _reserved :: Ptr{Cvoid} = C_NULL - float :: Ptr{Cvoid} = C_NULL # (o)->o - inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o - floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - index :: Ptr{Cvoid} = C_NULL # (o)->o - matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o -end - -@kwdef struct PySequenceMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - _was_item :: Ptr{Cvoid} = C_NULL - ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint - _was_ass_slice :: Ptr{Cvoid} = C_NULL - contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint - inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o -end - -@kwdef struct PyMappingMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o - ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint -end - -@kwdef struct PyBufferProcs - get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint - release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid -end - -@kwdef struct Py_buffer - buf :: Ptr{Cvoid} = C_NULL - obj :: Ptr{Cvoid} = C_NULL - len :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - readonly :: Cint = 0 - ndim :: Cint = 0 - format :: Cstring = C_NULL - shape :: Ptr{Py_ssize_t} = C_NULL - strides :: Ptr{Py_ssize_t} = C_NULL - suboffsets :: Ptr{Py_ssize_t} = C_NULL - internal :: Ptr{Cvoid} = C_NULL -end - -@kwdef struct PyTypeObject - ob_base :: PyVarObject = PyVarObject() - name :: Cstring = C_NULL - - basicsize :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - - dealloc :: Ptr{Cvoid} = C_NULL - vectorcall_offset :: Py_ssize_t = 0 - getattr :: Ptr{Cvoid} = C_NULL - setattr :: Ptr{Cvoid} = C_NULL - as_async :: Ptr{Cvoid} = C_NULL - repr :: Ptr{Cvoid} = C_NULL - - as_number :: Ptr{PyNumberMethods} = C_NULL - as_sequence :: Ptr{PySequenceMethods} = C_NULL - as_mapping :: Ptr{PyMappingMethods} = C_NULL - - hash :: Ptr{Cvoid} = C_NULL - call :: Ptr{Cvoid} = C_NULL - str :: Ptr{Cvoid} = C_NULL - getattro :: Ptr{Cvoid} = C_NULL - setattro :: Ptr{Cvoid} = C_NULL - - as_buffer :: Ptr{PyBufferProcs} = C_NULL - - flags :: Culong = 0 - - doc :: Cstring = C_NULL - - traverse :: Ptr{Cvoid} = C_NULL - - clear :: Ptr{Cvoid} = C_NULL - - richcompare :: Ptr{Cvoid} = C_NULL - - weaklistoffset :: Py_ssize_t = 0 - - iter :: Ptr{Cvoid} = C_NULL - iternext :: Ptr{Cvoid} = C_NULL - - methods :: Ptr{PyMethodDef} = C_NULL - members :: Ptr{PyMemberDef} = C_NULL - getset :: Ptr{PyGetSetDef} = C_NULL - base :: PyPtr = C_NULL - dict :: PyPtr = C_NULL - descr_get :: Ptr{Cvoid} = C_NULL - descr_set :: Ptr{Cvoid} = C_NULL - dictoffset :: Py_ssize_t = 0 - init :: Ptr{Cvoid} = C_NULL - alloc :: Ptr{Cvoid} = C_NULL - new :: Ptr{Cvoid} = C_NULL - free :: Ptr{Cvoid} = C_NULL - is_gc :: Ptr{Cvoid} = C_NULL - bases :: PyPtr = C_NULL - mro :: PyPtr = C_NULL - cache :: PyPtr = C_NULL - subclasses :: PyPtr = C_NULL - weaklist :: PyPtr = C_NULL - del :: Ptr{Cvoid} = C_NULL - - version_tag :: Cuint = 0 - - finalize :: Ptr{Cvoid} = C_NULL - vectorcall :: Ptr{Cvoid} = C_NULL - -end - -const PyTypePtr = Ptr{PyTypeObject} - pyglobal(name) = dlsym(CONFIG.libptr, name) -pyglobal(r::Ref{Ptr{Cvoid}}, name) = (p=r[]; if isnull(p); p=r[]=pyglobal(name); end; p) - +pyglobal(r::Ref{Ptr{T}}, name) where {T} = (p=r[]; if isnull(p); p=r[]=Ptr{T}(pyglobal(name)); end; p) pyloadglobal(r::Ref{Ptr{T}}, name) where {T} = (p=r[]; if isnull(p); p=r[]=unsafe_load(Ptr{Ptr{T}}(pyglobal(name))) end; p) macro cdef(name, rettype, argtypes) @@ -344,677 +25,100 @@ macro cdef(name, rettype, argtypes) end end -### INITIALIZE - -@cdef :Py_Initialize Cvoid () -@cdef :Py_InitializeEx Cvoid (Cint,) -@cdef :Py_Finalize Cvoid () -@cdef :Py_FinalizeEx Cint () -@cdef :Py_AtExit Cint (Ptr{Cvoid},) -@cdef :Py_IsInitialized Cint () - -@cdef :Py_SetPythonHome Cvoid (Cwstring,) -@cdef :Py_SetProgramName Cvoid (Cwstring,) -@cdef :Py_GetVersion Cstring () - -### REFCOUNT - -@cdef :Py_IncRef Cvoid (PyPtr,) -@cdef :Py_DecRef Cvoid (PyPtr,) -Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] - -Py_DecRef(f::Function, o::Ptr, dflt=PYERR()) = - isnull(o) ? dflt : (r=f(o); Py_DecRef(o); r) - -Py_Is(o1, o2) = Base.unsafe_convert(PyPtr, o1) == Base.unsafe_convert(PyPtr, o2) - -### EVAL - -@cdef :PyEval_EvalCode PyPtr (PyPtr, PyPtr, PyPtr) -@cdef :Py_CompileString PyPtr (Cstring, Cstring, Cint) -@cdef :PyEval_GetBuiltins PyPtr () - -### GIL & THREADS - -@cdef :PyEval_SaveThread Ptr{Cvoid} () -@cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) -@cdef :PyGILState_Ensure PyGILState_STATE () -@cdef :PyGILState_Release Cvoid (PyGILState_STATE,) - -### IMPORT - -@cdef :PyImport_ImportModule PyPtr (Cstring,) -@cdef :PyImport_Import PyPtr (PyPtr,) - -### ERRORS - -@cdef :PyErr_Occurred PyPtr () -@cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) -@cdef :PyErr_Clear Cvoid () -@cdef :PyErr_SetNone Cvoid (PyPtr,) -@cdef :PyErr_SetString Cvoid (PyPtr, Cstring) -@cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) -@cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) -@cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) -@cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) - -PyErr_IsSet() = !isnull(PyErr_Occurred()) - -PyErr_IsSet(t) = (o=PyErr_Occurred(); !isnull(o) && PyErr_GivenExceptionMatches(o,t)!=0) - -function PyErr_FetchTuple(normalize::Bool=false) - t = Ref{PyPtr}() - v = Ref{PyPtr}() - b = Ref{PyPtr}() - PyErr_Fetch(t, v, b) - normalize && PyErr_NormalizeException(t, v, b) - (t[], v[], b[]) -end - -### EXCEPTIONS - -for x in [:BaseException, :Exception, :StopIteration, :GeneratorExit, :ArithmeticError, - :LookupError, :AssertionError, :AttributeError, :BufferError, :EOFError, - :FloatingPointError, :OSError, :ImportError, :IndexError, :KeyError, :KeyboardInterrupt, - :MemoryError, :NameError, :OverflowError, :RuntimeError, :RecursionError, - :NotImplementedError, :SyntaxError, :IndentationError, :TabError, :ReferenceError, - :SystemError, :SystemExit, :TypeError, :UnboundLocalError, :UnicodeError, - :UnicodeEncodeError, :UnicodeDecodeError, :UnicodeTranslateError, :ValueError, - :ZeroDivisionError, :BlockingIOError, :BrokenPipeError, :ChildProcessError, - :ConnectionError, :ConnectionAbortedError, :ConnectionRefusedError, :FileExistsError, - :FileNotFoundError, :InterruptedError, :IsADirectoryError, :NotADirectoryError, - :PermissionError, :ProcessLookupError, :TimeoutError, :EnvironmentError, :IOError, - :WindowsError, :Warning, :UserWarning, :DeprecationWarning, :PendingDeprecationWarning, - :SyntaxWarning, :RuntimeWarning, :FutureWarning, :ImportWarning, :UnicodeWarning, - :BytesWarning, :ResourceWarning] - f = Symbol(:PyExc_, x) - r = Symbol(f, :__ref) - @eval const $r = Ref(PyPtr()) - @eval $f() = pyloadglobal($r, $(QuoteNode(f))) -end - -### NONE - -const Py_None__ref = Ref(C_NULL) -Py_None() = PyPtr(pyglobal(Py_None__ref, :_Py_NoneStruct)) - -PyNone_Check(o) = Py_Is(o, Py_None()) - -PyNone_New() = (o=Py_None(); Py_IncRef(o); o) - -PyNone_As(o, ::Type{T}) where {T} = - if Nothing <: T - nothing - elseif Missing <: T - missing - else - NOTIMPLEMENTED() - end - -### OBJECT - -@cdef :_PyObject_New PyPtr (PyPtr,) -@cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) -@cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) -@cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) -@cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) -@cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) -@cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) -@cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) -@cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) -@cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) -@cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) -@cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) -@cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) -@cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) -@cdef :PyObject_Repr PyPtr (PyPtr,) -@cdef :PyObject_ASCII PyPtr (PyPtr,) -@cdef :PyObject_Str PyPtr (PyPtr,) -@cdef :PyObject_Bytes PyPtr (PyPtr,) -@cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) -@cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) -@cdef :PyObject_Hash Py_hash_t (PyPtr,) -@cdef :PyObject_IsTrue Cint (PyPtr,) -@cdef :PyObject_Length Py_ssize_t (PyPtr,) -@cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) -@cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) -@cdef :PyObject_DelItem Cint (PyPtr, PyPtr) -@cdef :PyObject_Dir PyPtr (PyPtr,) -@cdef :PyObject_GetIter PyPtr (PyPtr,) -@cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) -@cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) - -function PyObject_ReprAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} - x = PyObject_Repr(o) - isnull(x) && return PYERR() - r = PyUnicode_As(x, T) - Py_DecRef(x) - r -end - -function PyObject_StrAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} - x = PyObject_Str(o) - isnull(x) && return PYERR() - r = PyUnicode_As(x, T) - Py_DecRef(x) - r -end - -function PyObject_ASCIIAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} - x = PyObject_ASCII(o) - isnull(x) && return PYERR() - r = PyUnicode_As(x, T) - Py_DecRef(x) - r -end - -function PyObject_BytesAs(o, ::Type{T}) where {T<:Union{Vector{UInt8},Vector{Int8},String}} - x = PyObject_Bytes(o) - isnull(x) && return PYERR() - r = PyBytes_As(x, T) - Py_DecRef(x) - r -end - -PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) -PyObject_From(x::Nothing) = PyNone_New() -PyObject_From(x::Bool) = PyBool_From(x) -PyObject_From(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = PyLong_From(x) -PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) -PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) -PyObject_From(x::Tuple) = PyTuple_From(x) -function PyObject_From(x) - PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") - PyPtr() -end - -function PyObject_CallArgs(f, args, kwargs=()) - if !isempty(kwargs) - error("kwargs not implemented") - elseif !isempty(args) - argso = PyTuple_From(Tuple(args)) - isnull(argso) && return PyPtr() - r = PyObject_CallObject(f, argso) - Py_DecRef(argso) - return r - else - PyObject_CallObject(f, C_NULL) - end -end - -PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) - -const PYOBJECT_AS_RULES = Dict{PyPtr, Union{Nothing,Function}}() -const PYOBJECT_AS_CRULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() -const PYOBJECT_AS_CRULES_CACHE = IdDict{Type, Dict{PyPtr, Any}}() - -function PyObject_As__rule(t::PyPtr) - name = Py_DecRef(PyObject_GetAttrString(t, "__name__")) do tnameo - Py_DecRef(PyObject_GetAttrString(t, "__module__")) do mnameo - tname = PyUnicode_As(tnameo, String) - tname === PYERR() && return PYERR() - mname = PyUnicode_As(mnameo, String) - mname === PYERR() && return PYERR() - "$mname.$tname" - end - end - name == PYERR() && return PYERR() - PyObject_As__rule(t, Val(Symbol(name))) -end - -PyObject_As__rule(t::PyPtr, ::Val) = nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.NoneType")}) = Py_Is(t, Py_Type(Py_None())) ? PyNone_As : nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.bool")}) = Py_Is(t, PyBool_Type()) ? PyBool_As : nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.str")}) = Py_Is(t, PyUnicode_Type()) ? PyUnicode_As : nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.bytes")}) = Py_Is(t, PyBytes_Type()) ? PyBytes_As : nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.int")}) = Py_Is(t, PyLong_Type()) ? PyLong_As : nothing -PyObject_As__rule(t::PyPtr, ::Val{Symbol("builtins.float")}) = Py_Is(t, PyFloat_Type()) ? PyFloat_As : nothing - -struct PyObject_As__crule_struct{T,F} - f :: F -end -function (r::PyObject_As__crule_struct{T,F})(o::PyPtr, ref::Base.RefValue{Any}) where {T,F} - res = r.f(o, T) - if res === PYERR() - return Cint(2) - elseif res === NOTIMPLEMENTED() - return Cint(1) - else - ref[] = res - return Cint(0) - end -end - -function PyObject_As(o::PyPtr, ::Type{T}) where {T} - # Run through the MRO, applying conversion rules that depend on the supertypes of the type of o. - # For speed, the conversion functions are cached as C function pointers. - # These take as inputs the PyPtr o and a Ptr{Any} in which to store the result. - # They return 0 on success, 1 on no-conversion and 2 on error. - mro = PyType_MRO(Py_Type(o)) - ref = Ref{Any}() - rules = get!(Dict{PyPtr, Ptr{Cvoid}}, PYOBJECT_AS_CRULES, T)::Dict{PyPtr,Ptr{Cvoid}} - for i in 1:PyTuple_Size(mro) - t = PyTuple_GetItem(mro, i-1) - isnull(t) && return PYERR() - crule = get(rules, t, missing) - if crule === missing - rule = PyObject_As__rule(t) - rule === PYERR() && return PYERR() - if rule === nothing - rules[t] = C_NULL - continue - else - crulefunc = @cfunction($(PyObject_As__crule_struct{T, typeof(rule)}(rule)), Cint, (PyPtr, Any)) - get!(Dict{PyPtr, Any}, PYOBJECT_AS_CRULES_CACHE, T)[t] = crulefunc - crule = rules[t] = Base.unsafe_convert(Ptr{Cvoid}, crulefunc) - end - elseif crule == C_NULL - continue - end - res = ccall(crule, Cint, (PyPtr, Any), o, ref) - if res == 0 - return ref[]::T - elseif res == 2 - return PYERR() - end - end - NOTIMPLEMENTED() -end -PyObject_As(o, ::Type{T}) where {T} = GC.@preserve o PyObject_As(Base.unsafe_convert(PyPtr, o), T) - -### SEQUENCE - -@cdef :PySequence_Length Py_ssize_t (PyPtr,) -@cdef :PySequence_GetItem PyPtr (PyPtr, Py_ssize_t) -@cdef :PySequence_Contains Cint (PyPtr, PyPtr) - -### MAPPING - -@cdef :PyMapping_HasKeyString Cint (PyPtr, Cstring) -@cdef :PyMapping_SetItemString Cint (PyPtr, Cstring, PyPtr) -@cdef :PyMapping_GetItemString PyPtr (PyPtr, Cstring) - -PyMapping_ExtractOneAs(o, k, ::Type{T}) where {T} = - Py_DecRef(PyMapping_GetItemString(o, string(k))) do x - v = PyObject_As(x, T) - if v === NOTIMPLEMENTED() - PyErr_SetString(PyExc_TypeError(), "Cannot convert this '$(PyType_Name(Py_Type(x)))' at key '$k' to a Julia '$T'") - PYERR() - else - v - end - end - -PyMapping_ExtractAs(o::PyPtr, ::Type{NamedTuple{names,types}}) where {names, types} = begin - t = PyMapping_ExtractAs(o, names, types) - t === PYERR() ? PYERR() : NamedTuple{names,types}(t) -end -PyMapping_ExtractAs(o::PyPtr, names::Tuple, ::Type{types}) where {types<:Tuple} = begin - v = PyMapping_ExtractOneAs(o, first(names), Base.tuple_type_head(types)) - v === PYERR() && return PYERR() - vs = PyMapping_ExtractAs(o::PyPtr, Base.tail(names), Base.tuple_type_tail(types)) - vs === PYERR() && return PYERR() - (v, vs...) -end -PyMapping_ExtractAs(o::PyPtr, names::Tuple{}, ::Type{Tuple{}}) = () -PyMapping_ExtractAs(o, ::Type{T}) where {T} = GC.@preserve o PyMapping_ExtractAs(Base.unsafe_convert(PyPtr, o), T) - -### METHOD - -@cdef :PyInstanceMethod_New PyPtr (PyPtr,) - -### STR - -@cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) -@cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) - -const PyUnicode_Type__ref = Ref(C_NULL) -PyUnicode_Type() = PyPtr(pyglobal(PyUnicode_Type__ref, :PyUnicode_Type)) - -PyUnicode_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_UNICODE_SUBCLASS) -PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) - -PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = - PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) - -PyUnicode_As(o, ::Type{T}) where {T} = begin - if (S = _typeintersect(T, AbstractString)) != Union{} - r = Py_DecRef(PyUnicode_AsUTF8String(o)) do b - PyBytes_As(b, S) - end - r === NOTIMPLEMENTED() || return r - end - if Symbol <: T - s = PyUnicode_As(o, String) - s === PYERR() && return PYERR() - s isa String && return Symbol(s) - end - if (S = _typeintersect(T, AbstractChar)) != Union{} - s = PyUnicode_As(o, String) - s === PYERR() && return PYERR() - if s isa String - if length(s) == 1 - c = first(s) - r = tryconvert(S, c) - r === NOTIMPLEMENTED() || return r - end - end - end - if (S = _typeintersect(T, AbstractVector)) != Union{} - r = Py_DecRef(PyUnicode_AsUTF8String(o)) do b - PyBytes_As(b, S) - end - r === NOTIMPLEMENTED() || return r - end - NOTIMPLEMENTED() -end - -### BYTES - -@cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) -@cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) - -PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = - PyBytes_FromStringAndSize(pointer(s), sizeof(s)) - -PyBytes_As(o, ::Type{T}) where {T} = begin - if (S = _typeintersect(T, AbstractVector{UInt8})) != Union{} - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return PYERR() - v = copy(Base.unsafe_wrap(Vector{UInt8}, Ptr{UInt8}(ptr[]), len[])) - r = tryconvert(S, v) - r === NOTIMPLEMENTED() || return r - end - if (S = _typeintersect(T, AbstractVector{Int8})) != Union{} - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return PYERR() - v = copy(Base.unsafe_wrap(Vector{Int8}, Ptr{Int8}(ptr[]), len[])) - r = tryconvert(S, v) - r === NOTIMPLEMENTED() || return r - end - if (S = _typeintersect(T, AbstractString)) != Union{} - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return PYERR() - s = Base.unsafe_string(ptr[], len[]) - r = tryconvert(S, s) - r === NOTIMPLEMENTED() || return r - end - NOTIMPLEMENTED() -end - -### TUPLE - -@cdef :PyTuple_New PyPtr (Py_ssize_t,) -@cdef :PyTuple_Size Py_ssize_t (PyPtr,) -@cdef :PyTuple_GetItem PyPtr (PyPtr, Py_ssize_t) -@cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) - -function PyTuple_From(xs::Tuple) - t = PyTuple_New(length(xs)) - isnull(t) && return PyPtr() - for (i,x) in enumerate(xs) - xo = PyObject_From(x) - isnull(xo) && (Py_DecRef(t); return PyPtr()) - err = PyTuple_SetItem(t, i-1, xo) # steals xo - ism1(err) && (Py_DecRef(t); return PyPtr()) - end - return t -end - -### TYPE - -@cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) -@cdef :PyType_Ready Cint (PyPtr,) - -Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] -Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) -Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) -Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) - -PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] -PyType_Name(o) = GC.@preserve o unsafe_string(UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).name[!]) -PyType_MRO(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).mro[!] - -PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) -PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) - -const PyType_Type__ref = Ref(C_NULL) -PyType_Type() = PyPtr(pyglobal(PyType_Type__ref, :PyType_Type)) - -PyType_Check(o) = Py_TypeCheck(o, Py_TPFLAGS_TYPE_SUBCLASS) - -PyType_CheckExact(o) = Py_TypeCheckExact(o, PyType_Type()) - -### NUMBER - -@cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) -@cdef :PyNumber_Negative PyPtr (PyPtr,) -@cdef :PyNumber_Positive PyPtr (PyPtr,) -@cdef :PyNumber_Absolute PyPtr (PyPtr,) -@cdef :PyNumber_Invert PyPtr (PyPtr,) -@cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_And PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) -@cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) -@cdef :PyNumber_Long PyPtr (PyPtr,) -@cdef :PyNumber_Float PyPtr (PyPtr,) -@cdef :PyNumber_Index PyPtr (PyPtr,) - -### ITER - -@cdef :PyIter_Next PyPtr (PyPtr,) - -### BOOL - -const PyBool_Type__ref = Ref(C_NULL) -PyBool_Type() = PyPtr(pyglobal(PyBool_Type__ref, :PyBool_Type)) - -const Py_True__ref = Ref(C_NULL) -Py_True() = PyPtr(pyglobal(Py_True__ref, :_Py_TrueStruct)) - -const Py_False__ref = Ref(C_NULL) -Py_False() = PyPtr(pyglobal(Py_False__ref, :_Py_FalseStruct)) - -PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) - -PyBool_Check(o) = Py_TypeCheckExact(o, PyBool_Type()) - -PyBool_As(o, ::Type{T}) where {T} = - if Bool <: T - if Py_Is(o, Py_True()) - true - elseif Py_Is(o, Py_False()) - false - else - PyErr_SetString(PyExc_TypeError(), "not a 'bool'") - PYERR() - end - else - NOTIMPLEMENTED() - end - - -### INT - -@cdef :PyLong_FromLongLong PyPtr (Clonglong,) -@cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) -@cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) -@cdef :PyLong_AsLongLong Clonglong (PyPtr,) -@cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) - -const PyLong_Type__ref = Ref(C_NULL) -PyLong_Type() = PyPtr(pyglobal(PyLong_Type__ref, :PyLong_Type)) - -PyLong_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LONG_SUBCLASS) - -PyLong_CheckExact(o) = Py_TypeCheckExact(o, PyLong_Type()) - -PyLong_From(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) = - if x isa Signed && typemin(Clonglong) ≤ x ≤ typemax(Clonglong) - PyLong_FromLongLong(x) - elseif typemin(Culonglong) ≤ x ≤ typemax(Culonglong) - PyLong_FromUnsignedLongLong(x) - else - PyLong_FromString(string(x), C_NULL, 10) - end - -PyLong_From(x::Integer) = begin - y = tryconvert(BigInt, x) - y === PYERR() && return PyPtr() - y === NOTIMPLEMENTED() && (PyErr_SetString(PyExc_NotImplementedError(), "Cannot convert this Julia '$(typeof(x))' to a Python 'int'"); return PyPtr()) - PyLong_From(y::BigInt) -end - -PyLong_As(o, ::Type{T}) where {T} = begin - if (S = _typeintersect(T, Integer)) != Union{} - # first try to convert to Clonglong (or Culonglong if unsigned) - x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) - if !ism1(x) || !PyErr_IsSet() - # success - return tryconvert(S, x) - elseif PyErr_IsSet(PyExc_OverflowError()) - # overflows Clonglong or Culonglong - PyErr_Clear() - if S in (Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128) && typemin(typeof(x)) ≤ typemin(S) && typemax(S) ≤ typemax(typeof(x)) - # definitely overflows S, give up now - return NOTIMPLEMENTED() - else - # try converting to String then BigInt then S - s = PyObject_StrAs(o, String) - s === PYERR() && return PYERR() - y = tryparse(BigInt, s) - y === nothing && (PyErr_SetString(PyExc_ValueError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia 'BigInt' because its string representation cannot be parsed as an integer"); return PYERR()) - return tryconvert(S, y::BigInt) - end - else - # other error - return PYERR() - end - elseif (S = _typeintersect(T, Real)) != Union{} - return tryconvert(S, PyLong_As(o, Integer)) - elseif (S = _typeintersect(T, Number)) != Union{} - return tryconvert(S, PyLong_As(o, Integer)) - else - return tryconvert(T, PyLong_As(o, Integer)) - end - NOTIMPLEMENTED() -end - -### FLOAT - -@cdef :PyFloat_FromDouble PyPtr (Cdouble,) -@cdef :PyFloat_AsDouble Cdouble (PyPtr,) - -const PyFloat_Type__ref = Ref(C_NULL) -PyFloat_Type() = PyPtr(pyglobal(PyFloat_Type__ref, :PyFloat_Type)) - -PyFloat_Check(o) = Py_TypeCheck(o, PyFloat_Type()) - -PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) - -PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) - -PyFloat_As(o, ::Type{T}) where {T} = begin - x = PyFloat_AsDouble(o) - ism1(x) && PyErr_IsSet() && return PYERR() - if Float64 <: T - return convert(Float64, x) - elseif Float32 <: T - return convert(Float32, x) - elseif Float16 <: T - return convert(Float16, x) - elseif (S = _typeintersect(T, AbstractFloat)) != Union{} - return tryconvert(S, x) - elseif (S = _typeintersect(T, Real)) != Union{} - return tryconvert(S, x) - elseif (S = _typeintersect(T, Number)) != Union{} - return tryconvert(S, x) - else - return tryconvert(T, x) - end -end - -### COMPLEX - -@cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) -@cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) - -### LIST - -@cdef :PyList_New PyPtr (Py_ssize_t,) -@cdef :PyList_Append Cint (PyPtr, PyPtr) -@cdef :PyList_AsTuple PyPtr (PyPtr,) - -### DICT - -@cdef :PyDict_New PyPtr () -@cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) -@cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) -@cdef :PyDict_DelItemString Cint (PyPtr, Cstring) - -### BUFFER - -function PyObject_CheckBuffer(o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - !isnull(p) && !isnull(p.get[]) -end - -function PyObject_GetBuffer(o, b, flags) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if isnull(p) || isnull(p.get[]) - PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") - return Cint(-1) - end - ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) -end - -function PyBuffer_Release(_b) - b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) - o = b.obj[] - isnull(o) && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (!isnull(p) && !isnull(p.release[])) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) - end - b.obj[] = C_NULL - Py_DecRef(o) - return -end - -### INPUT HOOK - -function PyOS_RunInputHook() - hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) - isnull(hook) || ccall(hook, Cint, ()) - nothing +include("consts.jl") +include("fundamentals.jl") +include("none.jl") +include("object.jl") +include("sequence.jl") +include("mapping.jl") +include("method.jl") +include("str.jl") +include("bytes.jl") +include("tuple.jl") +include("type.jl") +include("number.jl") +include("iter.jl") +include("bool.jl") +include("int.jl") +include("float.jl") +include("complex.jl") +include("list.jl") +include("dict.jl") +include("set.jl") +include("buffer.jl") +include("collections.jl") + +__init__() = begin + PyObject_TryConvert_AddRules("builtins.NoneType", [ + (Nothing, PyNone_TryConvertRule_nothing, 100), + (Missing, PyNone_TryConvertRule_missing), + ]) + PyObject_TryConvert_AddRules("builtins.bool", [ + (Bool, PyBool_TryConvertRule_bool, 100), + ]) + PyObject_TryConvert_AddRules("builtins.int", [ + (Integer, PyLong_TryConvertRule_integer, 100), + (Rational, PyLong_TryConvertRule_tryconvert), + (Real, PyLong_TryConvertRule_tryconvert), + (Number, PyLong_TryConvertRule_tryconvert), + (Any, PyLong_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.float", [ + (Float64, PyFloat_TryConvertRule_convert, 100), + (BigFloat, PyFloat_TryConvertRule_convert), + (Float32, PyFloat_TryConvertRule_convert), + (Float16, PyFloat_TryConvertRule_convert), + (AbstractFloat, PyFloat_TryConvertRule_tryconvert), + (Real, PyFloat_TryConvertRule_tryconvert), + (Number, PyFloat_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.complex", [ + (Complex{Float64}, PyComplex_TryConvertRule_convert, 100), + (Complex{BigFloat}, PyComplex_TryConvertRule_convert), + (Complex{Float32}, PyComplex_TryConvertRule_convert), + (Complex{Float16}, PyComplex_TryConvertRule_convert), + (Complex{T} where {T<:AbstractFloat}, PyComplex_TryConvertRule_tryconvert), + (Complex{T} where {T<:Real}, PyComplex_TryConvertRule_tryconvert), + (Number, PyComplex_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.bytes", [ + (Vector{UInt8}, PyBytes_TryConvertRule_vector), + (Vector{Int8}, PyBytes_TryConvertRule_vector), + (String, PyBytes_TryConvertRule_string), + ]) + PyObject_TryConvert_AddRules("builtins.str", [ + (String, PyUnicode_TryConvertRule_string, 100), + (Symbol, PyUnicode_TryConvertRule_symbol), + (Char, PyUnicode_TryConvertRule_char), + (Vector{UInt8}, PyUnicode_TryConvertRule_vector), + (Vector{Int8}, PyUnicode_TryConvertRule_vector), + ]) + PyObject_TryConvert_AddRules("builtins.tuple", [ + (Tuple, PyIterable_ConvertRule_tuple, 100), + ]) + PyObject_TryConvert_AddRules("collections.abc.Iterable", [ + (Vector, PyIterable_ConvertRule_vector), + (Set, PyIterable_ConvertRule_set), + (Tuple, PyIterable_ConvertRule_tuple), + (Pair, PyIterable_ConvertRule_pair), + ]) + PyObject_TryConvert_AddRules("collections.abc.Sequence", [ + (Vector, PyIterable_ConvertRule_vector), + ]) + PyObject_TryConvert_AddRules("collections.abc.Set", [ + (Set, PyIterable_ConvertRule_set), + ]) + PyObject_TryConvert_AddRules("collections.abc.Mapping", [ + (Dict, PyMapping_ConvertRule_dict), + ]) + PyObject_TryConvert_AddExtraTypes([ + ("collections.abc.Iterable", PyIterableABC_SubclassCheck), + ("collections.abc.Callable", PyCallableABC_SubclassCheck), + ("collections.abc.Sequence", PySequenceABC_SubclassCheck), + ("collections.abc.Mapping", PyMappingABC_SubclassCheck), + ("collections.abc.Set", PySetABC_SubclassCheck), + ("builtins.bufferable", PyType_CheckBuffer), + ]) end end diff --git a/src/cpython/bool.jl b/src/cpython/bool.jl new file mode 100644 index 00000000..4990352b --- /dev/null +++ b/src/cpython/bool.jl @@ -0,0 +1,22 @@ +const PyBool_Type__ref = Ref(PyPtr()) +PyBool_Type() = pyglobal(PyBool_Type__ref, :PyBool_Type) + +const Py_True__ref = Ref(PyPtr()) +Py_True() = pyglobal(Py_True__ref, :_Py_TrueStruct) + +const Py_False__ref = Ref(PyPtr()) +Py_False() = pyglobal(Py_False__ref, :_Py_FalseStruct) + +PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) + +PyBool_Check(o) = Py_TypeCheckExact(o, PyBool_Type()) + +PyBool_TryConvertRule_bool(o, ::Type{T}, ::Type{Bool}) where {T} = + if Py_Is(o, Py_True()) + putresult(T, true) + elseif Py_Is(o, Py_False()) + putresult(T, false) + else + PyErr_SetString(PyExc_TypeError(), "Expecting a 'bool' but got a '$(PyType_Name(Py_Type(o)))'") + -1 + end diff --git a/src/cpython/buffer.jl b/src/cpython/buffer.jl new file mode 100644 index 00000000..8e7c94d9 --- /dev/null +++ b/src/cpython/buffer.jl @@ -0,0 +1,28 @@ +PyType_CheckBuffer(t) = begin + p = UnsafePtr{PyTypeObject}(t).as_buffer[] + !isnull(p) && !isnull(p.get[!]) +end + +PyObject_CheckBuffer(o) = PyType_CheckBuffer(Py_Type(o)) + +PyObject_GetBuffer(o, b, flags) = begin + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if isnull(p) || isnull(p.get[]) + PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") + return Cint(-1) + end + ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) +end + +PyBuffer_Release(_b) = begin + b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) + o = b.obj[] + isnull(o) && return + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if (!isnull(p) && !isnull(p.release[])) + ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) + end + b.obj[] = C_NULL + Py_DecRef(o) + return +end diff --git a/src/cpython/bytes.jl b/src/cpython/bytes.jl new file mode 100644 index 00000000..6cb1356c --- /dev/null +++ b/src/cpython/bytes.jl @@ -0,0 +1,29 @@ +@cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) +@cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) + +const PyBytes_Type__ref = Ref(PyPtr()) +PyBytes_Type() = pyglobal(PyBytes_Type__ref, :PyBytes_Type) + +PyBytes_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_BYTES_SUBCLASS) +PyBytes_CheckExact(o) = Py_TypeCheckExact(o, PyBytes_Type()) + +PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyBytes_FromStringAndSize(pointer(s), sizeof(s)) + +PyBytes_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T,X} = begin + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return -1 + v = copy(Base.unsafe_wrap(Vector{X}, Ptr{X}(ptr[]), len[])) + return putresult(T, v) +end + +PyBytes_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return -1 + v = Base.unsafe_string(ptr[], len[]) + return putresult(T, v) +end diff --git a/src/cpython/collections.jl b/src/cpython/collections.jl new file mode 100644 index 00000000..b7c1586e --- /dev/null +++ b/src/cpython/collections.jl @@ -0,0 +1,216 @@ +for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, :Sized, + :Callable, :Collection, :Sequence, :MutableSequence, :ByteString, :Set, :MutableSet, + :Mapping, :MutableMapping, :MappingView, :ItemsView, :KeysView, :ValuesView, + :Awaitable, :Coroutine, :AsyncIterable, :AsyncIterator, :AsyncGenerator] + p = Symbol(:Py, n, :ABC) + t = Symbol(p, :_Type) + tr = Symbol(p, :__ref) + c = Symbol(p, :_Check) + ct = Symbol(p, :_SubclassCheck) + @eval const $tr = Ref(PyPtr()) + @eval $t() = begin + ptr = $tr[] + isnull(ptr) || return ptr + a = PyImport_ImportModule("collections.abc") + isnull(a) && return a + b = PyObject_GetAttrString(a, $(string(n))) + Py_DecRef(a) + isnull(b) && return b + $tr[] = b + end + @eval $c(_o) = GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + t = $t() + isnull(t) && return Cint(-1) + PyObject_IsInstance(o, t) + end + @eval $ct(_o) = GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + t = $t() + isnull(t) && return Cint(-1) + PyObject_IsSubclass(o, t) + end +end + +PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + xo = PyIter_Next(it) + if !isnull(xo) + r = PyObject_TryConvert(xo, eltype(xs)) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(eltype(xs)) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(T, xs) + end + end +end +PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{Vector}) where {T} = + PyIterable_ConvertRule_vector(o, T, Vector{Python.PyObject}) + +PyIterable_ConvertRule_set(o, ::Type{T}, ::Type{S}) where {T, S<:Set} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + xo = PyIter_Next(it) + if !isnull(xo) + r = PyObject_TryConvert(xo, eltype(xs)) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(eltype(xs)) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(T, xs) + end + end +end +PyIterable_ConvertRule_set(o, ::Type{T}, ::Type{Set}) where {T} = + PyIterable_ConvertRule_set(o, T, Set{Python.PyObject}) + +PyIterable_ConvertRule_tuple(o, ::Type{T}, ::Type{S}) where {T,S<:Tuple} = begin + if !(Tuple isa DataType) + PyErr_SetString(PyExc_Exception(), "When converting Python 'tuple' to Julia 'Tuple', the destination type must be a 'DataType', i.e. not parametric and not a union. Got '$S'.") + return -1 + end + ts = S.parameters + if !isempty(ts) && Base.isvarargtype(ts[end]) + isvararg = true + vartype = ts[end].body.parameters[1] + ts = ts[1:end-1] + else + isvararg = false + end + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = Union{ts..., isvararg ? vartype : Union{}}[] + i = 0 + while true + xo = PyIter_Next(it) + if !isnull(xo) + i += 1 + if i ≤ length(ts) + t = ts[i] + elseif isvararg + t = vartype + else + Py_DecRef(it) + Py_DecRef(xo) + return -1 + end + r = PyObject_TryConvert(xo, t) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(t) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(T, S(xs)) + end + end +end +PyIterable_ConvertRule_tuple(o, ::Type{T}, ::Type{Tuple{}}) where {T} = putresult(T, ()) + +PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K,V}}) where {T,K,V} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + # get the first item + ko = PyIter_Next(it) + if isnull(ko) + Py_DecRef(it) + return PyErr_IsSet() ? -1 : 0 + end + # convert it + r = PyObject_TryConvert(ko, K) + Py_DecRef(ko) + if r != 1 + Py_DecRef(it) + return r + end + k = takeresult(K) + # get the second item + vo = PyIter_Next(it) + if isnull(vo) + Py_DecRef(it) + return PyErr_IsSet() ? -1 : 0 + end + # convert it + r = PyObject_TryConvert(vo, V) + Py_DecRef(vo) + if r != 1 + Py_DecRef(it) + return r + end + v = takeresult(V) + # too many values? + xo = PyIter_Next(it) + if !isnull(xo) + Py_DecRef(xo) + Py_DecRef(it) + return 0 + end + # done + Py_DecRef(it) + putresult(T, Pair{K,V}(k, v)) +end +PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K}}) where {T,K} = + PyIterable_ConvertRule_pair(o, T, Pair{K,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K,V} where K}) where {T,V} = + PyIterable_ConvertRule_pair(o, T, Pair{Python.PyObject,V}) +PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair}) where {T} = + PyIterable_ConvertRule_pair(o, T, Pair{Python.PyObject,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{S}) where {T,S<:Pair} = begin + PyErr_SetString(PyExc_Exception(), "When converting Python iterable to Julia 'Pair', the destination type cannot be too complicated: the two types must either be fully specified or left unspecified. Got '$S'.") + return -1 +end + +PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{S}) where {T, S<:Dict} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + ko = PyIter_Next(it) + if !isnull(ko) + # get the key + r = PyObject_TryConvert(ko, keytype(xs)) + r == 1 || (Py_DecRef(it); Py_DecRef(ko); return r) + k = takeresult(keytype(xs)) + # get the value + vo = PyObject_GetItem(o, ko) + isnull(vo) && (Py_DecRef(it); Py_DecRef(ko); return -1) + r = PyObject_TryConvert(vo, valtype(xs)) + Py_DecRef(vo) + Py_DecRef(ko) + r == 1 || (Py_DecRef(it); return -1) + v = takeresult(valtype(xs)) + xs[k] = v + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(T, xs) + end + end +end +PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict{K}}) where {T,K} = + PyMapping_ConvertRule_dict(o, T, Dict{K,Python.PyObject}) +PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict{K,V} where K}) where {T,V} = + PyMapping_ConvertRule_dict(o, T, Dict{Python.PyObject,V}) +PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict}) where {T} = + PyMapping_ConvertRule_dict(o, T, Dict{Python.PyObject,Python.PyObject}) diff --git a/src/cpython/complex.jl b/src/cpython/complex.jl new file mode 100644 index 00000000..a35ab534 --- /dev/null +++ b/src/cpython/complex.jl @@ -0,0 +1,31 @@ +@cdef :PyComplex_FromDoubles PyPtr (Cdouble, Cdouble) +@cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) +@cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) + +const PyComplex_Type__ref = Ref(PyPtr()) +PyComplex_Type() = pyglobal(PyComplex_Type__ref, :PyComplex_Type) + +PyComplex_Check(o) = Py_TypeCheck(o, PyComplex_Type()) + +PyComplex_CheckExact(o) = Py_TypeCheckExact(o, PyComplex_Type()) + +PyComplex_From(x::Union{Float16,Float32,Float64}) = PyComplex_FromDoubles(x, 0) +PyComplex_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_FromDoubles(real(x), imag(x)) + +PyComplex_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyComplex_RealAsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + y = PyComplex_ImagAsDouble(o) + ism1(y) && PyErr_IsSet() && return -1 + z = Complex(x, y) + putresult(T, convert(S, z)) +end + +PyComplex_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyComplex_RealAsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + y = PyComplex_ImagAsDouble(o) + ism1(y) && PyErr_IsSet() && return -1 + z = Complex(x, y) + putresult(T, tryconvert(S, z)) +end diff --git a/src/cpython/consts.jl b/src/cpython/consts.jl new file mode 100644 index 00000000..36cc2334 --- /dev/null +++ b/src/cpython/consts.jl @@ -0,0 +1,309 @@ +@enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 + +const Py_single_input = 256 +const Py_file_input = 257 +const Py_eval_input = 258 +const Py_func_type_input = 345 + +const Py_LT = Cint(0) +const Py_LE = Cint(1) +const Py_EQ = Cint(2) +const Py_NE = Cint(3) +const Py_GT = Cint(4) +const Py_GE = Cint(5) + +const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments +const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs +const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) +const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) +const Py_METH_CLASS = 0x0010 # for class methods +const Py_METH_STATIC = 0x0020 # for static methods + +const Py_T_SHORT =0 +const Py_T_INT =1 +const Py_T_LONG =2 +const Py_T_FLOAT =3 +const Py_T_DOUBLE =4 +const Py_T_STRING =5 +const Py_T_OBJECT =6 +const Py_T_CHAR =7 +const Py_T_BYTE =8 +const Py_T_UBYTE =9 +const Py_T_USHORT =10 +const Py_T_UINT =11 +const Py_T_ULONG =12 +const Py_T_STRING_INPLACE =13 +const Py_T_BOOL =14 +const Py_T_OBJECT_EX =16 +const Py_T_LONGLONG =17 # added in Python 2.5 +const Py_T_ULONGLONG =18 # added in Python 2.5 +const Py_T_PYSSIZET =19 # added in Python 2.6 +const Py_T_NONE =20 # added in Python 3.0 + +const Py_READONLY = 1 +const Py_READ_RESTRICTED = 2 +const Py_WRITE_RESTRICTED = 4 +const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) + +const PyBUF_MAX_NDIM = 64 + +# Flags for getting buffers +const PyBUF_SIMPLE = 0x0 +const PyBUF_WRITABLE = 0x0001 +const PyBUF_WRITEABLE = PyBUF_WRITABLE +const PyBUF_FORMAT = 0x0004 +const PyBUF_ND = 0x0008 +const PyBUF_STRIDES = (0x0010 | PyBUF_ND) +const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) +const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) +const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) +const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) + +const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) +const PyBUF_CONTIG_RO = (PyBUF_ND) + +const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) +const PyBUF_STRIDED_RO = (PyBUF_STRIDES) + +const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) + +const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) + +const PyBUF_READ = 0x100 +const PyBUF_WRITE = 0x200 + +# Python 2.7 +const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) +const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) +const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 +const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) +const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) +const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) +const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) +const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) +const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) +const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) +const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) +const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) + +# Python 3.0+ has only these: +const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) +const Py_TPFLAGS_BASETYPE = (0x00000001<<10) +const Py_TPFLAGS_READY = (0x00000001<<12) +const Py_TPFLAGS_READYING = (0x00000001<<13) +const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) +const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) +const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) +const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) +const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) +const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) +const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) +const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) +const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) +const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) +const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) +const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) +const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) + +# only use this if we have the stackless extension +const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) + +const Py_hash_t = Cssize_t +const Py_ssize_t = Cssize_t + +@kwdef struct Py_complex + real :: Cdouble = 0.0 + imag :: Cdouble = 0.0 +end + +@kwdef struct PyObject + # assumes _PyObject_HEAD_EXTRA is empty + refcnt :: Py_ssize_t = 0 + type :: Ptr{PyObject} = C_NULL +end + +const PyPtr = Ptr{PyObject} + +struct PyObjectRef + ptr :: PyPtr +end +ispyreftype(::Type{PyObjectRef}) = true +pyptr(o::PyObjectRef) = o.ptr +Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr + +@kwdef struct PyVarObject + ob_base :: PyObject = PyObject() + size :: Py_ssize_t = 0 +end + +@kwdef struct PyMethodDef + name :: Cstring = C_NULL + meth :: Ptr{Cvoid} = C_NULL + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyGetSetDef + name :: Cstring = C_NULL + get :: Ptr{Cvoid} = C_NULL + set :: Ptr{Cvoid} = C_NULL + doc :: Cstring = C_NULL + closure :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyMemberDef + name :: Cstring = C_NULL + typ :: Cint = 0 + offset :: Py_ssize_t = 0 + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyNumberMethods + add :: Ptr{Cvoid} = C_NULL # (o,o)->o + subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o + power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + negative :: Ptr{Cvoid} = C_NULL # (o)->o + positive :: Ptr{Cvoid} = C_NULL # (o)->o + absolute :: Ptr{Cvoid} = C_NULL # (o)->o + bool :: Ptr{Cvoid} = C_NULL # (o)->Cint + invert :: Ptr{Cvoid} = C_NULL # (o)->o + lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + and :: Ptr{Cvoid} = C_NULL # (o,o)->o + xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + or :: Ptr{Cvoid} = C_NULL # (o,o)->o + int :: Ptr{Cvoid} = C_NULL # (o)->o + _reserved :: Ptr{Cvoid} = C_NULL + float :: Ptr{Cvoid} = C_NULL # (o)->o + inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o + floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + index :: Ptr{Cvoid} = C_NULL # (o)->o + matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o +end + +@kwdef struct PySequenceMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + _was_item :: Ptr{Cvoid} = C_NULL + ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint + _was_ass_slice :: Ptr{Cvoid} = C_NULL + contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint + inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o +end + +@kwdef struct PyMappingMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o + ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint +end + +@kwdef struct PyBufferProcs + get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint + release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid +end + +@kwdef struct Py_buffer + buf :: Ptr{Cvoid} = C_NULL + obj :: Ptr{Cvoid} = C_NULL + len :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + readonly :: Cint = 0 + ndim :: Cint = 0 + format :: Cstring = C_NULL + shape :: Ptr{Py_ssize_t} = C_NULL + strides :: Ptr{Py_ssize_t} = C_NULL + suboffsets :: Ptr{Py_ssize_t} = C_NULL + internal :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyTypeObject + ob_base :: PyVarObject = PyVarObject() + name :: Cstring = C_NULL + + basicsize :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + + dealloc :: Ptr{Cvoid} = C_NULL + vectorcall_offset :: Py_ssize_t = 0 + getattr :: Ptr{Cvoid} = C_NULL + setattr :: Ptr{Cvoid} = C_NULL + as_async :: Ptr{Cvoid} = C_NULL + repr :: Ptr{Cvoid} = C_NULL + + as_number :: Ptr{PyNumberMethods} = C_NULL + as_sequence :: Ptr{PySequenceMethods} = C_NULL + as_mapping :: Ptr{PyMappingMethods} = C_NULL + + hash :: Ptr{Cvoid} = C_NULL + call :: Ptr{Cvoid} = C_NULL + str :: Ptr{Cvoid} = C_NULL + getattro :: Ptr{Cvoid} = C_NULL + setattro :: Ptr{Cvoid} = C_NULL + + as_buffer :: Ptr{PyBufferProcs} = C_NULL + + flags :: Culong = 0 + + doc :: Cstring = C_NULL + + traverse :: Ptr{Cvoid} = C_NULL + + clear :: Ptr{Cvoid} = C_NULL + + richcompare :: Ptr{Cvoid} = C_NULL + + weaklistoffset :: Py_ssize_t = 0 + + iter :: Ptr{Cvoid} = C_NULL + iternext :: Ptr{Cvoid} = C_NULL + + methods :: Ptr{PyMethodDef} = C_NULL + members :: Ptr{PyMemberDef} = C_NULL + getset :: Ptr{PyGetSetDef} = C_NULL + base :: PyPtr = C_NULL + dict :: PyPtr = C_NULL + descr_get :: Ptr{Cvoid} = C_NULL + descr_set :: Ptr{Cvoid} = C_NULL + dictoffset :: Py_ssize_t = 0 + init :: Ptr{Cvoid} = C_NULL + alloc :: Ptr{Cvoid} = C_NULL + new :: Ptr{Cvoid} = C_NULL + free :: Ptr{Cvoid} = C_NULL + is_gc :: Ptr{Cvoid} = C_NULL + bases :: PyPtr = C_NULL + mro :: PyPtr = C_NULL + cache :: PyPtr = C_NULL + subclasses :: PyPtr = C_NULL + weaklist :: PyPtr = C_NULL + del :: Ptr{Cvoid} = C_NULL + + version_tag :: Cuint = 0 + + finalize :: Ptr{Cvoid} = C_NULL + vectorcall :: Ptr{Cvoid} = C_NULL + +end + +const PyTypePtr = Ptr{PyTypeObject} diff --git a/src/cpython/dict.jl b/src/cpython/dict.jl new file mode 100644 index 00000000..2e1c0d14 --- /dev/null +++ b/src/cpython/dict.jl @@ -0,0 +1,47 @@ +@cdef :PyDict_New PyPtr () +@cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyDict_DelItemString Cint (PyPtr, Cstring) + +const PyDict_Type__ref = Ref(PyPtr()) +PyDict_Type() = pyglobal(PyDict_Type__ref, :PyDict_Type) + +PyDict_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_DICT_SUBCLASS) +PyDict_CheckExact(o) = Py_TypeCheckExact(o, PyDict_Type()) + +PyDict_FromPairs(kvs) = begin + r = PyDict_New() + isnull(r) && return PyPtr() + for (k,v) in kvs + ko = PyObject_From(k) + isnull(ko) && (Py_DecRef(r); return PyPtr()) + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); Py_DecRef(ko); return PyPtr()) + err = PyDict_SetItem(r, ko, vo) + Py_DecRef(ko) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + r +end + +PyDict_FromStringPairs(kvs) = begin + r = PyDict_New() + isnull(r) && return PyPtr() + for (k,v) in kvs + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); return PyPtr()) + err = PyDict_SetItemString(r, string(k), vo) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + r +end + +PyDict_From(x::Dict) = PyDict_FromPairs(x) +PyDict_From(x::Dict{String}) = PyDict_FromStringPairs(x) +PyDict_From(x::Dict{Symbol}) = PyDict_FromStringPairs(x) +PyDict_From(x::NamedTuple) = PyDict_FromStringPairs(pairs(x)) +PyDict_From(x::Base.Iterators.Pairs) = PyDict_FromPairs(x) +PyDict_From(x::Base.Iterators.Pairs{String}) = PyDict_FromStringPairs(x) +PyDict_From(x::Base.Iterators.Pairs{Symbol}) = PyDict_FromStringPairs(x) diff --git a/src/cpython/error.jl b/src/cpython/error.jl new file mode 100644 index 00000000..e69de29b diff --git a/src/cpython/float.jl b/src/cpython/float.jl new file mode 100644 index 00000000..72c77092 --- /dev/null +++ b/src/cpython/float.jl @@ -0,0 +1,23 @@ +@cdef :PyFloat_FromDouble PyPtr (Cdouble,) +@cdef :PyFloat_AsDouble Cdouble (PyPtr,) + +const PyFloat_Type__ref = Ref(PyPtr()) +PyFloat_Type() = pyglobal(PyFloat_Type__ref, :PyFloat_Type) + +PyFloat_Check(o) = Py_TypeCheck(o, PyFloat_Type()) + +PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) + +PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) + +PyFloat_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyFloat_AsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(T, convert(S, x)) +end + +PyFloat_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyFloat_AsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(T, tryconvert(S, x)) +end diff --git a/src/cpython/fundamentals.jl b/src/cpython/fundamentals.jl new file mode 100644 index 00000000..fa873c68 --- /dev/null +++ b/src/cpython/fundamentals.jl @@ -0,0 +1,96 @@ +### INITIALIZE + +@cdef :Py_Initialize Cvoid () +@cdef :Py_InitializeEx Cvoid (Cint,) +@cdef :Py_Finalize Cvoid () +@cdef :Py_FinalizeEx Cint () +@cdef :Py_AtExit Cint (Ptr{Cvoid},) +@cdef :Py_IsInitialized Cint () + +@cdef :Py_SetPythonHome Cvoid (Cwstring,) +@cdef :Py_SetProgramName Cvoid (Cwstring,) +@cdef :Py_GetVersion Cstring () + +### REFCOUNT + +@cdef :Py_IncRef Cvoid (PyPtr,) +@cdef :Py_DecRef Cvoid (PyPtr,) +Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] + +Py_DecRef(f::Function, o::Ptr, dflt=PYERR()) = + isnull(o) ? dflt : (r=f(o); Py_DecRef(o); r) + +Py_Is(o1, o2) = Base.unsafe_convert(PyPtr, o1) == Base.unsafe_convert(PyPtr, o2) + +### EVAL + +@cdef :PyEval_EvalCode PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :Py_CompileString PyPtr (Cstring, Cstring, Cint) +@cdef :PyEval_GetBuiltins PyPtr () + +### GIL & THREADS + +@cdef :PyEval_SaveThread Ptr{Cvoid} () +@cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) +@cdef :PyGILState_Ensure PyGILState_STATE () +@cdef :PyGILState_Release Cvoid (PyGILState_STATE,) + +### IMPORT + +@cdef :PyImport_ImportModule PyPtr (Cstring,) +@cdef :PyImport_Import PyPtr (PyPtr,) + +### ERRORS + +@cdef :PyErr_Occurred PyPtr () +@cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) +@cdef :PyErr_Clear Cvoid () +@cdef :PyErr_SetNone Cvoid (PyPtr,) +@cdef :PyErr_SetString Cvoid (PyPtr, Cstring) +@cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) +@cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) + +PyErr_IsSet() = !isnull(PyErr_Occurred()) + +PyErr_IsSet(t) = (o=PyErr_Occurred(); !isnull(o) && PyErr_GivenExceptionMatches(o,t)!=0) + +function PyErr_FetchTuple(normalize::Bool=false) + t = Ref{PyPtr}() + v = Ref{PyPtr}() + b = Ref{PyPtr}() + PyErr_Fetch(t, v, b) + normalize && PyErr_NormalizeException(t, v, b) + (t[], v[], b[]) +end + +### EXCEPTIONS + +for x in [:BaseException, :Exception, :StopIteration, :GeneratorExit, :ArithmeticError, + :LookupError, :AssertionError, :AttributeError, :BufferError, :EOFError, + :FloatingPointError, :OSError, :ImportError, :IndexError, :KeyError, :KeyboardInterrupt, + :MemoryError, :NameError, :OverflowError, :RuntimeError, :RecursionError, + :NotImplementedError, :SyntaxError, :IndentationError, :TabError, :ReferenceError, + :SystemError, :SystemExit, :TypeError, :UnboundLocalError, :UnicodeError, + :UnicodeEncodeError, :UnicodeDecodeError, :UnicodeTranslateError, :ValueError, + :ZeroDivisionError, :BlockingIOError, :BrokenPipeError, :ChildProcessError, + :ConnectionError, :ConnectionAbortedError, :ConnectionRefusedError, :FileExistsError, + :FileNotFoundError, :InterruptedError, :IsADirectoryError, :NotADirectoryError, + :PermissionError, :ProcessLookupError, :TimeoutError, :EnvironmentError, :IOError, + :WindowsError, :Warning, :UserWarning, :DeprecationWarning, :PendingDeprecationWarning, + :SyntaxWarning, :RuntimeWarning, :FutureWarning, :ImportWarning, :UnicodeWarning, + :BytesWarning, :ResourceWarning] + f = Symbol(:PyExc_, x) + r = Symbol(f, :__ref) + @eval const $r = Ref(PyPtr()) + @eval $f() = pyloadglobal($r, $(QuoteNode(f))) +end + +### INPUT HOOK + +function PyOS_RunInputHook() + hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) + isnull(hook) || ccall(hook, Cint, ()) + nothing +end diff --git a/src/cpython/int.jl b/src/cpython/int.jl new file mode 100644 index 00000000..8e25cb8a --- /dev/null +++ b/src/cpython/int.jl @@ -0,0 +1,62 @@ +@cdef :PyLong_FromLongLong PyPtr (Clonglong,) +@cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) +@cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) +@cdef :PyLong_AsLongLong Clonglong (PyPtr,) +@cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) + +const PyLong_Type__ref = Ref(PyPtr()) +PyLong_Type() = pyglobal(PyLong_Type__ref, :PyLong_Type) + +PyLong_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LONG_SUBCLASS) + +PyLong_CheckExact(o) = Py_TypeCheckExact(o, PyLong_Type()) + +PyLong_From(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) = + if x isa Signed && typemin(Clonglong) ≤ x ≤ typemax(Clonglong) + PyLong_FromLongLong(x) + elseif typemin(Culonglong) ≤ x ≤ typemax(Culonglong) + PyLong_FromUnsignedLongLong(x) + else + PyLong_FromString(string(x), C_NULL, 10) + end + +PyLong_From(x::Integer) = begin + y = tryconvert(BigInt, x) + y === PYERR() && return PyPtr() + y === NOTIMPLEMENTED() && (PyErr_SetString(PyExc_NotImplementedError(), "Cannot convert this Julia '$(typeof(x))' to a Python 'int'"); return PyPtr()) + PyLong_From(y::BigInt) +end + +PyLong_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} = begin + # first try to convert to Clonglong (or Culonglong if unsigned) + x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) + if !ism1(x) || !PyErr_IsSet() + # success + return putresult(T, tryconvert(S, x)) + elseif PyErr_IsSet(PyExc_OverflowError()) + # overflows Clonglong or Culonglong + PyErr_Clear() + if S in (Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128) && typemin(typeof(x)) ≤ typemin(S) && typemax(S) ≤ typemax(typeof(x)) + # definitely overflows S, give up now + return 0 + else + # try converting to String then BigInt then S + so = PyObject_Str(o) + isnull(so) && return -1 + r = PyUnicode_TryConvertRule_string(so, String, String) + Py_DecRef(so) + r == 1 || return r + y = tryparse(BigInt, takeresult(String)) + y === nothing && (PyErr_SetString(PyExc_ValueError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia 'BigInt' because its string representation cannot be parsed as an integer"); return -1) + return putresult(T, tryconvert(S, y)) + end + else + # other error + return -1 + end +end + +PyLong_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + r = PyLong_TryConvertRule_integer(o, Integer, Integer) + r == 1 ? putresult(T, tryconvert(S, takeresult(Integer))) : r +end diff --git a/src/cpython/iter.jl b/src/cpython/iter.jl new file mode 100644 index 00000000..84c6d74d --- /dev/null +++ b/src/cpython/iter.jl @@ -0,0 +1 @@ +@cdef :PyIter_Next PyPtr (PyPtr,) diff --git a/src/cpython/list.jl b/src/cpython/list.jl new file mode 100644 index 00000000..56409c81 --- /dev/null +++ b/src/cpython/list.jl @@ -0,0 +1,3 @@ +@cdef :PyList_New PyPtr (Py_ssize_t,) +@cdef :PyList_Append Cint (PyPtr, PyPtr) +@cdef :PyList_AsTuple PyPtr (PyPtr,) diff --git a/src/cpython/mapping.jl b/src/cpython/mapping.jl new file mode 100644 index 00000000..f7e0b985 --- /dev/null +++ b/src/cpython/mapping.jl @@ -0,0 +1,3 @@ +@cdef :PyMapping_HasKeyString Cint (PyPtr, Cstring) +@cdef :PyMapping_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyMapping_GetItemString PyPtr (PyPtr, Cstring) diff --git a/src/cpython/method.jl b/src/cpython/method.jl new file mode 100644 index 00000000..4b1605f6 --- /dev/null +++ b/src/cpython/method.jl @@ -0,0 +1 @@ +@cdef :PyInstanceMethod_New PyPtr (PyPtr,) diff --git a/src/cpython/none.jl b/src/cpython/none.jl new file mode 100644 index 00000000..c5760202 --- /dev/null +++ b/src/cpython/none.jl @@ -0,0 +1,9 @@ +const Py_None__ref = Ref(PyPtr()) +Py_None() = pyglobal(Py_None__ref, :_Py_NoneStruct) + +PyNone_Check(o) = Py_Is(o, Py_None()) + +PyNone_New() = (o=Py_None(); Py_IncRef(o); o) + +PyNone_TryConvertRule_nothing(o, ::Type{T}, ::Type{Nothing}) where {T} = putresult(T, nothing) +PyNone_TryConvertRule_missing(o, ::Type{T}, ::Type{Missing}) where {T} = putresult(T, missing) diff --git a/src/cpython/number.jl b/src/cpython/number.jl new file mode 100644 index 00000000..6b2937f5 --- /dev/null +++ b/src/cpython/number.jl @@ -0,0 +1,34 @@ +@cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_Negative PyPtr (PyPtr,) +@cdef :PyNumber_Positive PyPtr (PyPtr,) +@cdef :PyNumber_Absolute PyPtr (PyPtr,) +@cdef :PyNumber_Invert PyPtr (PyPtr,) +@cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_And PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Long PyPtr (PyPtr,) +@cdef :PyNumber_Float PyPtr (PyPtr,) +@cdef :PyNumber_Index PyPtr (PyPtr,) diff --git a/src/cpython/object.jl b/src/cpython/object.jl new file mode 100644 index 00000000..7862ed3b --- /dev/null +++ b/src/cpython/object.jl @@ -0,0 +1,236 @@ +@cdef :_PyObject_New PyPtr (PyPtr,) +@cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) +@cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) +@cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) +@cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) +@cdef :PyObject_Repr PyPtr (PyPtr,) +@cdef :PyObject_ASCII PyPtr (PyPtr,) +@cdef :PyObject_Str PyPtr (PyPtr,) +@cdef :PyObject_Bytes PyPtr (PyPtr,) +@cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) +@cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) +@cdef :PyObject_Hash Py_hash_t (PyPtr,) +@cdef :PyObject_IsTrue Cint (PyPtr,) +@cdef :PyObject_Length Py_ssize_t (PyPtr,) +@cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelItem Cint (PyPtr, PyPtr) +@cdef :PyObject_Dir PyPtr (PyPtr,) +@cdef :PyObject_GetIter PyPtr (PyPtr,) +@cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) + +PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) +PyObject_From(x::Nothing) = PyNone_New() +PyObject_From(x::Bool) = PyBool_From(x) +PyObject_From(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = PyLong_From(x) +PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) +PyObject_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_From(x) +PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) +PyObject_From(x::Tuple) = PyTuple_From(x) +PyObject_From(x::T) where {T} = + if ispyreftype(T) + GC.@preserve x begin + ptr = pyptr(x) + Py_IncRef(ptr) + ptr + end + else + PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") + PyPtr() + end + +function PyObject_CallArgs(f, args::Tuple, kwargs::Union{Nothing,NamedTuple,Base.Iterators.Pairs{Symbol},Base.Iterators.Pairs{Union{}}}=nothing) + if kwargs!==nothing && !isempty(kwargs) + argso = PyTuple_From(args) + isnull(argso) && return PyPtr() + kwargso = PyDict_From(kwargs) + isnull(kwargso) && (Py_DecRef(argso); return PyPtr()) + r = PyObject_Call(f, argso, kwargso) + Py_DecRef(argso) + Py_DecRef(kwargso) + r + elseif !isempty(args) + argso = PyTuple_From(Tuple(args)) + isnull(argso) && return PyPtr() + r = PyObject_CallObject(f, argso) + Py_DecRef(argso) + return r + else + PyObject_CallObject(f, C_NULL) + end +end + +PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) + +# const PYCONVERT_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() +# const PYCONVERT_RULES_CACHE = IdDict{Type, Dict{PyPtr, Any}}() + +# @generated PyConvert_GetRules(::Type{T}) where {T} = +# get!(Dict{PyPtr, Ptr{Cvoid}}, PYCONVERT_RULES, T) + +# function PyObject_Convert__rule(t::PyPtr) +# name = Py_DecRef(PyObject_GetAttrString(t, "__name__")) do tnameo +# Py_DecRef(PyObject_GetAttrString(t, "__module__")) do mnameo +# r = PyUnicode_TryConvert(tnameo, String) +# r == 1 || return PYERR() +# tname = takeresult(String) +# r = PyUnicode_TryConvert(mnameo, String) +# r == 1 || return PYERR() +# mname = takeresult(String) +# "$mname.$tname" +# end +# end +# name == PYERR() && return PYERR() +# PyObject_Convert__rule(t, Val(Symbol(name))) +# end + +# PyObject_Convert__rule(t::PyPtr, ::Val) = nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.NoneType")}) = Py_Is(t, Py_Type(Py_None())) ? PyNone_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.bool")}) = Py_Is(t, PyBool_Type()) ? PyBool_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.str")}) = Py_Is(t, PyUnicode_Type()) ? PyUnicode_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.bytes")}) = Py_Is(t, PyBytes_Type()) ? PyBytes_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.int")}) = Py_Is(t, PyLong_Type()) ? PyLong_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.float")}) = Py_Is(t, PyFloat_Type()) ? PyFloat_TryConvert : nothing +# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.complex")}) = Py_Is(t, PyComplex_Type()) ? PyComplex_TryConvert : nothing + +# struct PyObject_Convert__rule_struct{T,F} +# f :: F +# end +# (r::PyObject_Convert__rule_struct{T,F})(o::PyPtr) where {T,F} = r.f(o, T)::Int + +const ERRPTR = Ptr{Cvoid}(1) + +const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() +const TRYCONVERT_RULES = Dict{String, Vector{Tuple{Int, Type, Function}}}() +const TRYCONVERT_EXTRATYPES = Vector{Tuple{String, Function}}() + +@generated PyObject_TryConvert_CompiledRules(::Type{T}) where {T} = + get!(Dict{PyPtr, Ptr{Cvoid}}, TRYCONVERT_COMPILED_RULES, T) +PyObject_TryConvert_Rules(n::String) = + get!(Vector{Tuple{Int, Type, Function}}, TRYCONVERT_RULES, n) +PyObject_TryConvert_AddRule(n::String, T::Type, rule::Function, priority::Int=0) = + push!(PyObject_TryConvert_Rules(n), (priority, T, rule)) +PyObject_TryConvert_AddRules(n::String, xs) = + for x in xs + PyObject_TryConvert_AddRule(n, x...) + end +PyObject_TryConvert_AddExtraType(n::String, pred::Function) = + pushfirst!(TRYCONVERT_EXTRATYPES, (n, pred)) +PyObject_TryConvert_AddExtraTypes(xs) = + for x in xs + PyObject_TryConvert_AddExtraType(x...) + end + +PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin + # first get the MRO + mrotuple = PyType_MRO(t) + mrotypes = PyPtr[] + mronames = String[] + for i in 1:PyTuple_Size(mrotuple) + b = PyTuple_GetItem(mrotuple, i-1) + isnull(b) && return ERRPTR + push!(mrotypes, b) + n = PyType_FullName(b) + n === PYERR() && return ERRPTR + push!(mronames, n) + end + # find any extra types + extranames = Dict{Int, Vector{String}}() + for (name, pred) in TRYCONVERT_EXTRATYPES + i = nothing + for (j,t) in enumerate(mrotypes) + r = pred(t) + if r === PYERR() + return ERRPTR + elseif r isa Bool + if r + i = j + end + elseif r isa Cint + r == -1 && return ERRPTR + if r != 0 + i = j + end + else + @warn "Unexpected return type from a Python convert extra-types predicate: `$(typeof(r))`" + end + end + i === nothing || push!(get!(Vector{String}, extranames, i), name) + end + # merge all the type names together + allnames = [n for (i,mroname) in enumerate(mronames) for n in [mroname; get(Vector{String}, extranames, i)]] + # gather rules of the form (priority, order, S, rule) from these types + rules = Tuple{Int, Int, Type, Function}[] + for n in allnames + for (p, S, r) in PyObject_TryConvert_Rules(n) + push!(rules, (p, length(rules)+1, S, r)) + end + end + # sort by priority + sort!(rules, by=x->(-x[1], x[2])) + # intersect S with T + rules = [typeintersect(S,T) => r for (p,i,S,r) in rules] + # discard rules where S is a subtype of the union of all previous S for rules with the same implementation + # in particular this removes rules with S==Union{} and removes duplicates + rules = [S=>rule for (i,(S,rule)) in enumerate(rules) if !(S <: Union{[S′ for (S′,rule′) in rules[1:i-1] if rule==rule′]...})] + println("CONVERSION RULES FOR '$(PyType_Name(t))' TO '$T':") + display(rules) + # make the function implementing these rules + rulefunc = @eval (o::PyPtr) -> begin + $((:(r = $rule(o, $T, $S)::Int; r == 0 || return r) for (S,rule) in rules)...) + return 0 + end + # compile it + rulecfunc = @cfunction($rulefunc, Int, (PyPtr,)) + push!(CACHE, rulecfunc) + Base.unsafe_convert(Ptr{Cvoid}, rulecfunc) +end + +PyObject_TryConvert(o::PyPtr, ::Type{T}) where {T} = begin + # First try based only on the type T + # Used mainly by wrapper types. + r = PyObject_TryConvert__initial(o, T) :: Int + r == 0 || return r + + # Try to find an appropriate conversion based on `T` and the type of `o`. + rules = PyObject_TryConvert_CompiledRules(T) + t = Py_Type(o) + rule = get(rules, t, ERRPTR) + if rule == ERRPTR + rule = PyObject_TryConvert_CompileRule(T, t) + rule == ERRPTR && return -1 + rules[t] = rule + end + if !isnull(rule) + r = ccall(rule, Int, (PyPtr,), o) + r == 0 || return r + end + + 0 +end +PyObject_TryConvert(o, ::Type{T}) where {T} = GC.@preserve o PyObject_TryConvert(Base.unsafe_convert(PyPtr, o), T) + +PyObject_TryConvert__initial(o, ::Type{T}) where {T} = 0 + +PyObject_Convert(o::PyPtr, ::Type{T}) where {T} = begin + r = PyObject_TryConvert(o, T) + if r == 0 + PyErr_SetString(PyExc_TypeError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia '$T'") + -1 + elseif r == -1 + -1 + else + 0 + end +end +PyObject_Convert(o, ::Type{T}) where {T} = GC.@preserve o PyObject_Convert(Base.unsafe_convert(PyPtr, o), T) diff --git a/src/cpython/sequence.jl b/src/cpython/sequence.jl new file mode 100644 index 00000000..414ddca5 --- /dev/null +++ b/src/cpython/sequence.jl @@ -0,0 +1,4 @@ +@cdef :PySequence_Length Py_ssize_t (PyPtr,) +@cdef :PySequence_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PySequence_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) +@cdef :PySequence_Contains Cint (PyPtr, PyPtr) diff --git a/src/cpython/set.jl b/src/cpython/set.jl new file mode 100644 index 00000000..d7083a51 --- /dev/null +++ b/src/cpython/set.jl @@ -0,0 +1,2 @@ +@cdef :PySet_New PyPtr (PyPtr,) +@cdef :PyFrozenSet_New PyPtr (PyPtr,) diff --git a/src/cpython/str.jl b/src/cpython/str.jl new file mode 100644 index 00000000..2ead90f7 --- /dev/null +++ b/src/cpython/str.jl @@ -0,0 +1,44 @@ +@cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) +@cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) +@cdef :PyUnicode_InternInPlace Cvoid (Ptr{PyPtr},) + +const PyUnicode_Type__ref = Ref(PyPtr()) +PyUnicode_Type() = pyglobal(PyUnicode_Type__ref, :PyUnicode_Type) + +PyUnicode_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_UNICODE_SUBCLASS) +PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) + +PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) + +PyUnicode_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin + b = PyUnicode_AsUTF8String(o) + isnull(b) && return -1 + r = PyBytes_TryConvertRule_string(b, String, String) + Py_DecRef(b) + r == 1 || return r + moveresult(String, T) +end + +PyUnicode_TryConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = begin + b = PyUnicode_AsUTF8String(o) + isnull(b) && return -1 + r = PyBytes_TryConvertRule_vector(b, S, S) + Py_DecRef(b) + r == 1 || return r + moveresult(S, T) +end + +PyUnicode_TryConvertRule_symbol(o, ::Type{T}, ::Type{Symbol}) where {T} = begin + r = PyUnicode_TryConvertRule_string(o, String, String) + r == 1 || return r + putresult(T, Symbol(takeresult(String))) +end + +PyUnicode_TryConvertRule_char(o, ::Type{T}, ::Type{Char}) where {T} = begin + r = PyUnicode_TryConvertRule_string(o, String, String) + r == 1 || return r + s = takeresult(String) + length(s) == 1 || return 0 + putresult(T, first(s)) +end diff --git a/src/cpython/tuple.jl b/src/cpython/tuple.jl new file mode 100644 index 00000000..b515b579 --- /dev/null +++ b/src/cpython/tuple.jl @@ -0,0 +1,16 @@ +@cdef :PyTuple_New PyPtr (Py_ssize_t,) +@cdef :PyTuple_Size Py_ssize_t (PyPtr,) +@cdef :PyTuple_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) + +function PyTuple_From(xs::Tuple) + t = PyTuple_New(length(xs)) + isnull(t) && return PyPtr() + for (i,x) in enumerate(xs) + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(t); return PyPtr()) + err = PyTuple_SetItem(t, i-1, xo) # steals xo + ism1(err) && (Py_DecRef(t); return PyPtr()) + end + return t +end diff --git a/src/cpython/type.jl b/src/cpython/type.jl new file mode 100644 index 00000000..0a723d2d --- /dev/null +++ b/src/cpython/type.jl @@ -0,0 +1,40 @@ +@cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) +@cdef :PyType_Ready Cint (PyPtr,) + +Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] +Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) +Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) +Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) + +PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] +PyType_Name(o) = GC.@preserve o unsafe_string(UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).name[!]) +PyType_MRO(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).mro[!] + +PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) +PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) + +const PyType_Type__ref = Ref(PyPtr()) +PyType_Type() = pyglobal(PyType_Type__ref, :PyType_Type) + +PyType_Check(o) = Py_TypeCheck(o, Py_TPFLAGS_TYPE_SUBCLASS) + +PyType_CheckExact(o) = Py_TypeCheckExact(o, PyType_Type()) + +PyType_FullName(o) = begin + # get __module__ + mo = PyObject_GetAttrString(o, "__module__") + isnull(mo) && return PYERR() + r = PyUnicode_TryConvertRule_string(mo, String, String) + Py_DecRef(mo) + r == 1 || return PYERR() + m = takeresult(String) + # get __qualname__ + no = PyObject_GetAttrString(o, "__qualname__") + isnull(no) && return PYERR() + r = PyUnicode_TryConvertRule_string(no, String, String) + Py_DecRef(no) + r == 1 || return PYERR() + n = takeresult(String) + # done + "$m.$n" +end diff --git a/src/eval.jl b/src/eval.jl index 6aef7f27..ae89228f 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,14 +1,6 @@ -module CompiledCode - import ..Python: PyCode - stash(co::PyCode) = begin - nm = gensym() - @eval $nm = $co - co - end - stash(args...) = stash(PyCode(args...)) -end +pyeval_filename(src) = isfile(string(src.file)) ? "$(src.file):$(src.line)" : "julia:$(src.file):$(src.line)" -pyeval_macro(src, mode, args...) = begin +pyeval_macro(filename, mode, args...) = begin # find the code argument icode = findfirst(args) do x x isa Expr && x.head == :macrocall && x.args[1] == :(`foo`).args[1] @@ -16,7 +8,7 @@ pyeval_macro(src, mode, args...) = begin icode in (1,2) || error() code = args[icode].args[end] # the return type - rettypearg = icode==2 ? args[1] : mode==:eval ? Any : Nothing + rettypearg = icode==2 ? args[1] : mode==:eval ? PyObject : Nothing # parse the remaining arguments - the first one is optionally the locals, the rest are var=value pairs to include in the locals kvs = [] locals = nothing @@ -53,7 +45,7 @@ pyeval_macro(src, mode, args...) = begin # currently only the pattern "^ $(...) =" is recognized as a LHS # TODO: multiple assignment # TODO: mutating assignment - islhs = [((match(r"\n\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"\s*=", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] + islhs = [((match(r"\n\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] # do the interpolation intvars = ["_jl_interp_$(i)_" for i in 1:length(interps)] for (k,v,lhs) in zip(intvars, interps, islhs) @@ -64,65 +56,76 @@ pyeval_macro(src, mode, args...) = begin if any(islhs) mode == :exec || error("interpolation on LHS only allowed in exec mode") end + if mode == :exec + outkvts = [(ex isa Expr && ex.head == :(::)) ? (esc(ex.args[1]), v, esc(ex.args[2])) : (esc(ex), v, PyObject) for (ex,v,lhs) in zip(interps,intvars,islhs) if lhs] + end # make the code object - co = CompiledCode.stash(newcode, "", mode) + co = PyCode(newcode, filename, mode) # go - freelocals = locals === nothing ? :(GC.@preserve locals C.Py_DecRef(lptr)) : nothing - quote - # get the code pointer - cptr = pyptr($co) - # get the globals pointer - globals = $(esc(:pyglobals)) - gptr = pyptr(globals) - # ensure globals includes builtins - if !globals.hasbuiltins - if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 - err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) - ism1(err) && pythrow() - end - globals.hasbuiltins = true - end - # get locals (ALLOCATES lptr if locals===nothing) - $(locals === nothing ? :(lptr = C.PyDict_New(); isnull(lptr) && pythrow()) : :(locals = $(esc(locals)); lptr = pyptr(locals))) - # insert extra locals - $([:(let; vo=C.PyObject_From($(esc(v))); isnull(vo) && ($freelocals; pythrow()); err=C.PyMapping_SetItemString(lptr, $(string(k)), vo); C.Py_DecRef(vo); ism1(err) && ($freelocals; pythrow()); end) for (k,v) in kvs]...) - # Call eval (ALLOCATES rptr) - rptr = GC.@preserve globals C.PyEval_EvalCode(cptr, gptr, lptr) - isnull(rptr) && ($freelocals; pythrow()) - # extract values - $( - if mode == :eval - quote - $freelocals - res = C.PyObject_As(rptr, $(esc(rettypearg))) - C.Py_DecRef(rptr) - res == PYERR() && pythrow() - res == NOTIMPLEMENTED() && (C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert return value of type '$(C.PyType_Name(C.Py_Type(rptr)))' to a Julia '$($(esc(rettypearg)))'"); pythrow()) - return res::$(esc(rettypearg)) - end - elseif mode == :exec - quote - C.Py_DecRef(rptr) - $((((jv,jt) = (ex isa Expr && ex.head == :(::)) ? (ex.args[1], esc(ex.args[2])) : (ex, Any); quote - $(esc(jv)) = let - xo = C.PyMapping_GetItemString(lptr, $v) - isnull(xo) && ($freelocals; pythrow()) - x = C.PyObject_As(xo, $jt) - x===NOTIMPLEMENTED() && C.PyErr_SetString(C.PyExc_TypeError(), "Cannot convert return value '$($(string(jv)))' of type '$(C.PyType_Name(C.Py_Type(xo)))' to a Julia '$($jt)'") - C.Py_DecRef(xo) - x===PYERR() && ($freelocals; pythrow()) - x===NOTIMPLEMENTED() && ($freelocals; pythrow()) - x::$jt; - end - end) for (ex,v,lhs) in zip(interps,intvars,islhs) if lhs)...) - $freelocals - nothing + freelocals = locals === nothing ? :(C.Py_DecRef(lptr)) : :(GC.@preserve locals nothing) + ret = quote + let + # evaluate the inputs (so any errors are thrown before we have object references) + $([:($(Symbol(:input,i)) = $(esc(v))) for (i,(k,v)) in enumerate(kvs)]...) + # get the code pointer + cptr = checknull(pyptr($co)) + # get the globals pointer + globals = $(esc(:pyglobals)) + gptr = checknull(pyptr(globals)) + # ensure globals includes builtins + if !globals.hasbuiltins + if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 + err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) + ism1(err) && pythrow() end - else - error() + globals.hasbuiltins = true end - ) + # get locals (ALLOCATES lptr if locals===nothing) + $(locals === nothing ? :(lptr = checknull(C.PyDict_New())) : :(locals = $(esc(locals)); lptr = checknull(pyptr(locals)))) + # insert extra locals + $([:(let; vo=C.PyObject_From($(Symbol(:input,i))); isnull(vo) && ($freelocals; pythrow()); err=C.PyObject_SetItem(lptr, $(PyInternedString(string(k))), vo); C.Py_DecRef(vo); ism1(err) && ($freelocals; pythrow()); end) for (i,(k,v)) in enumerate(kvs)]...) + # Call eval (ALLOCATES rptr) + rptr = GC.@preserve globals C.PyEval_EvalCode(cptr, gptr, lptr) + isnull(rptr) && ($freelocals; pythrow()) + # extract values + $( + if mode == :eval + quote + $freelocals + res = C.PyObject_Convert(rptr, $(esc(rettypearg))) + C.Py_DecRef(rptr) + ism1(res) && pythrow() + C.takeresult($(esc(rettypearg))) + end + elseif mode == :exec + quote + C.Py_DecRef(rptr) + $((quote + $(Symbol(:output,i)) = let + xo = C.PyObject_GetItem(lptr, $(PyInternedString(v))) + isnull(xo) && ($freelocals; pythrow()) + res = C.PyObject_Convert(xo, $t) + C.Py_DecRef(xo) + ism1(res) && ($freelocals; pythrow()) + C.takeresult($t) + end + end for (i,(_,v,t)) in enumerate(outkvts))...) + $freelocals + ($((Symbol(:output,i) for i in 1:length(outkvts))...),) + end + else + error() + end + ) + end end + if mode == :exec + ret = quote + ($((k for (k,_,_) in outkvts)...),) = $ret + nothing + end + end + ret end """ @@ -132,11 +135,12 @@ Executes the given Python code. Julia values can be interpolated using the usual `\$(...)` syntax. Additionally, assignment to interpolations is supported: e.g. `\$(x::T) = ...` will convert the right hand side to a `T` and assign it to `x`. +Currently only single assignment is supported, and it must occur at the start of a line; multiple assignment (`\$x, \$y = ...`) or mutating assignment (`\$x += ...`) will not be recognized. The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. """ macro py(args...) - pyeval_macro(__source__, :exec, args...) + pyeval_macro(pyeval_filename(__source__), :exec, args...) end export @py @@ -146,7 +150,7 @@ export @py Shorthand for ```@py `...` ```. """ macro py_str(code::String) - pyeval_macro(__source__, :exec, Expr(:macrocall, :(`foo`).args[1], code)) + pyeval_macro(pyeval_filename(__source__), :exec, Expr(:macrocall, :(`foo`).args[1], code)) end export @py_str @@ -162,10 +166,20 @@ The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temp The result is converted to a `rettype`, which defaults to `PyObject`. """ macro pyv(args...) - pyeval_macro(__source__, :eval, args...) + pyeval_macro(pyeval_filename(__source__), :eval, args...) end export @pyv +""" + pyv"..." + +Shorthand for ```@pyv `...` ```. +""" +macro pyv_str(code::String) + pyeval_macro(pyeval_filename(__source__), :eval, Expr(:macrocall, :(`foo`).args[1], code)) +end +export @pyv_str + """ py`...` :: PyCode @@ -174,7 +188,7 @@ A Python code object in "exec" mode which is compiled only once. Suitable for using as the `code` argument to `pyeval`. """ macro py_cmd(code::String) - CompiledCode.stash(code, "", :exec) + PyCode(code, pyeval_filename(__source__), :exec) end export @py_cmd @@ -186,6 +200,6 @@ A Python code object in "eval" mode which is compiled only once. Suitable for using as the `code` argument to `pyexec`. """ macro pyv_cmd(code::String) - CompiledCode.stash(code, "", :eval) + PyCode(code, pyeval_filename(__source__), :eval) end export @pyv_cmd diff --git a/src/init.jl b/src/init.jl index 2b728129..a2a781a5 100644 --- a/src/init.jl +++ b/src/init.jl @@ -122,6 +122,11 @@ function __init__() end end + C.PyObject_TryConvert_AddRule("builtins.object", PyObject, CTryConvertRule_wrapref, -100) + C.PyObject_TryConvert_AddRule("collections.abc.Sequence", PyList, CTryConvertRule_wrapref, 100) + C.PyObject_TryConvert_AddRule("collections.abc.Set", PySet, CTryConvertRule_wrapref, 100) + C.PyObject_TryConvert_AddRule("collections.abc.Mapping", PyDict, CTryConvertRule_wrapref, 100) + # with_gil() do # if !CONFIG.isembedded diff --git a/src/old/PyList.jl b/src/old/PyList.jl deleted file mode 100644 index 47cbfa2d..00000000 --- a/src/old/PyList.jl +++ /dev/null @@ -1,43 +0,0 @@ -""" - PyList{T=PyObject}(o=pylist()) - -Wrap the Python list `o` (or anything satisfying the sequence interface) as a Julia vector with elements of type `T`. -""" -struct PyList{T} <: AbstractVector{T} - o :: PyObject - PyList{T}(o::PyObject) where {T} = new{T}(o) -end -PyList{T}(o=pylist()) where {T} = PyList{T}(pylist(o)) -PyList(o=pylist()) = PyList{PyObject}(o) -export PyList - -pyobject(x::PyList) = x.o - -Base.length(x::PyList) = Int(pylen(x.o)) - -Base.size(x::PyList) = (length(x),) - -Base.getindex(x::PyList{T}, i::Integer) where {T} = pyconvert(T, pygetitem(x.o, i-1)) - -Base.setindex!(x::PyList{T}, v, i::Integer) where {T} = (pysetitem(x.o, i-1, convert(T, v)); x) - -Base.insert!(x::PyList{T}, i::Integer, v) where {T} = (x.o.insert(i-1, convert(T, v)); x) - -Base.push!(x::PyList{T}, v) where {T} = (x.o.append(convert(T, v)); x) - -Base.pushfirst!(x::PyList, v) = insert!(x, 1, v) - -Base.pop!(x::PyList{T}) where {T} = pyconvert(T, x.o.pop()) - -Base.popat!(x::PyList{T}, i::Integer) where {T} = pyconvert(T, x.o.pop(i-1)) - -Base.popfirst!(x::PyList) = pop!(x, 1) - -Base.reverse!(x::PyList) = (x.o.reverse(); x) - -# TODO: support kwarg `by` (becomes python kwarg `key`) -Base.sort!(x::PyList; rev=false) = (x.o.sort(reverse=rev); x) - -Base.empty!(x::PyList) = (x.o.clear(); x) - -Base.copy(x::PyList) = typeof(x)(x.o.copy()) diff --git a/src/old/PySet.jl b/src/old/PySet.jl deleted file mode 100644 index 8aa10122..00000000 --- a/src/old/PySet.jl +++ /dev/null @@ -1,73 +0,0 @@ -""" - PySet{T=PyObject}(o=pyset()) - -Wrap the Python set `o` (or anything satisfying the set interface) as a Julia set with elements of type `T`. -""" -struct PySet{T} <: AbstractSet{T} - o :: PyObject - PySet{T}(o::PyObject) where {T} = new{T}(o) -end -PySet{T}(o=pyset()) where {T} = PySet{T}(pyset(o)) -PySet(o=pyset()) = PySet{PyObject}(o) -export PySet - -pyobject(x::PySet) = x.o - -function Base.iterate(x::PySet{T}, it=pyiter(x.o)) where {T} - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing - else - (pyconvert(T, pynewobject(ptr)), it) - end -end - -Base.length(x::PySet) = Int(pylen(x.o)) - -function Base.in(_v, x::PySet{T}) where {T} - v = tryconvert(T, _v) - v === PyConvertFail() ? false : pycontains(x, v) -end - -function Base.push!(x::PySet{T}, v) where {T} - x.o.add(convert(T, v)) - x -end - -function Base.delete!(x::PySet{T}, _v) where {T} - v = tryconvert(T, _v) - v === PyConvertFail() || x.o.discard(v) - x -end - -function Base.pop!(x::PySet{T}) where {T} - v = x.o.pop() - pyconvert(T, v) -end - -function Base.pop!(x::PySet{T}, _v) where {T} - v = convert(T, _v) - x.o.remove(v) - v -end - -function Base.pop!(x::PySet{T}, _v, d) where {T} - v = tryconvert(T, _v) - (v !== PyConvertFail() && v in x) ? pop!(x, v) : d -end - -function Base.filter!(f, x::PySet{T}) where {T} - d = pylist() - for _v in x.o - if !f(pyconvert(T, _v)) - d.append(_v) - end - end - x.o.difference_update(d) - x -end - -Base.empty!(x::PySet) = (x.o.clear(); x) - -Base.copy(x::PySet) = typeof(x)(x.o.copy()) diff --git a/src/utils.jl b/src/utils.jl index 2cd4c113..5de76834 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -68,16 +68,39 @@ pybufferformat_to_type(fmt::AbstractString) = ### TYPE UTILITIES -struct PyConvertFail end +# Used to signal a Python error from functions that return general Julia objects +struct PYERR end -tryconvert(::Type{T}, x) where {T} = -try - convert(T, x) -catch - PyConvertFail() -end +# Used to signal that conversion failed +struct NOTIMPLEMENTED end + +# Somewhere to stash results +const RESULT = Ref{Any}(nothing) +_putresult(::Type{T}, x::T) where {T} = (RESULT[] = x) +takeresult(::Type{T}) where {T} = (r = RESULT[]::T; RESULT[] = nothing; r) + +const RESULT_INT = Ref{Int}() +_putresult(::Type{Int}, x::Int) = (RESULT_INT[] = x) +takeresult(::Type{Int}) = RESULT_INT[] + +putresult(::Type{T}, x::T) where {T} = (_putresult(T, x); 1) +putresult(::Type{T}, x::PYERR) where {T} = -1 +putresult(::Type{T}, x::NOTIMPLEMENTED) where {T} = 0 + +moveresult(::Type{T}, ::Type{T}) where {T} = 1 +moveresult(::Type{S}, ::Type{T}) where {S,T} = putresult(T, takeresult(S)) + +tryconvert(::Type{T}, x::PYERR) where {T} = PYERR() +tryconvert(::Type{T}, x::NOTIMPLEMENTED) where {T} = NOTIMPLEMENTED() tryconvert(::Type{T}, x::T) where {T} = x -tryconvert(::Type{T}, x::PyConvertFail) where {T} = x +tryconvert(::Type{T}, x) where {T} = + try + convert(T, x) + catch + NOTIMPLEMENTED() + end + +CTryConvertRule_wrapref(o, ::Type{T}, ::Type{S}) where {T,S} = putresult(T, S(C.PyObjectRef(o))) @generated _typeintersect(::Type{T1}, ::Type{T2}) where {T1,T2} = typeintersect(T1, T2) @@ -105,3 +128,6 @@ end isnull(p::Ptr) = Ptr{Cvoid}(p) == C_NULL isnull(p::UnsafePtr) = isnull(pointer(p)) ism1(x::T) where {T<:Number} = x == (zero(T)-one(T)) + +# Put something in here to keep it around forever +const CACHE = [] From ac0a297f6e6540b441e8a70520fcf4c2bbac5ca4 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Dec 2020 18:29:04 +0000 Subject: [PATCH 07/14] remove debugging printing --- src/cpython/object.jl | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/src/cpython/object.jl b/src/cpython/object.jl index 7862ed3b..f9290271 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -72,42 +72,6 @@ end PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) -# const PYCONVERT_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() -# const PYCONVERT_RULES_CACHE = IdDict{Type, Dict{PyPtr, Any}}() - -# @generated PyConvert_GetRules(::Type{T}) where {T} = -# get!(Dict{PyPtr, Ptr{Cvoid}}, PYCONVERT_RULES, T) - -# function PyObject_Convert__rule(t::PyPtr) -# name = Py_DecRef(PyObject_GetAttrString(t, "__name__")) do tnameo -# Py_DecRef(PyObject_GetAttrString(t, "__module__")) do mnameo -# r = PyUnicode_TryConvert(tnameo, String) -# r == 1 || return PYERR() -# tname = takeresult(String) -# r = PyUnicode_TryConvert(mnameo, String) -# r == 1 || return PYERR() -# mname = takeresult(String) -# "$mname.$tname" -# end -# end -# name == PYERR() && return PYERR() -# PyObject_Convert__rule(t, Val(Symbol(name))) -# end - -# PyObject_Convert__rule(t::PyPtr, ::Val) = nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.NoneType")}) = Py_Is(t, Py_Type(Py_None())) ? PyNone_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.bool")}) = Py_Is(t, PyBool_Type()) ? PyBool_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.str")}) = Py_Is(t, PyUnicode_Type()) ? PyUnicode_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.bytes")}) = Py_Is(t, PyBytes_Type()) ? PyBytes_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.int")}) = Py_Is(t, PyLong_Type()) ? PyLong_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.float")}) = Py_Is(t, PyFloat_Type()) ? PyFloat_TryConvert : nothing -# PyObject_Convert__rule(t::PyPtr, ::Val{Symbol("builtins.complex")}) = Py_Is(t, PyComplex_Type()) ? PyComplex_TryConvert : nothing - -# struct PyObject_Convert__rule_struct{T,F} -# f :: F -# end -# (r::PyObject_Convert__rule_struct{T,F})(o::PyPtr) where {T,F} = r.f(o, T)::Int - const ERRPTR = Ptr{Cvoid}(1) const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() @@ -183,8 +147,8 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin # discard rules where S is a subtype of the union of all previous S for rules with the same implementation # in particular this removes rules with S==Union{} and removes duplicates rules = [S=>rule for (i,(S,rule)) in enumerate(rules) if !(S <: Union{[S′ for (S′,rule′) in rules[1:i-1] if rule==rule′]...})] - println("CONVERSION RULES FOR '$(PyType_Name(t))' TO '$T':") - display(rules) + # println("CONVERSION RULES FOR '$(PyType_Name(t))' TO '$T':") + # display(rules) # make the function implementing these rules rulefunc = @eval (o::PyPtr) -> begin $((:(r = $rule(o, $T, $S)::Int; r == 0 || return r) for (S,rule) in rules)...) From ac1af401fb6fc3e0169a7a1d79ef4bde91cf813c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 18 Dec 2020 12:02:52 +0000 Subject: [PATCH 08/14] more conversions --- src/PyException.jl | 33 +++++------ src/builtins.jl | 2 +- src/cpython/CPython.jl | 87 +++++++++++++++++++--------- src/cpython/collections.jl | 10 +--- src/cpython/complex.jl | 25 ++++---- src/cpython/float.jl | 5 +- src/cpython/int.jl | 5 +- src/cpython/number.jl | 23 ++++++++ src/cpython/object.jl | 113 ++++++++++++++++++++++++------------- src/cpython/range.jl | 90 +++++++++++++++++++++++++++++ src/cpython/type.jl | 5 ++ src/eval.jl | 2 +- src/init.jl | 1 + 13 files changed, 289 insertions(+), 112 deletions(-) create mode 100644 src/cpython/range.jl diff --git a/src/PyException.jl b/src/PyException.jl index fcca36a5..c1daf413 100644 --- a/src/PyException.jl +++ b/src/PyException.jl @@ -140,28 +140,21 @@ function Base.showerror(io::IO, e::PyException) try @py ``` import traceback - $(fs :: C.PyObjectRef) = fs = traceback.extract_tb($(e.bref)) - $(nfs :: Int) = len(fs) + $(fs :: Vector{Tuple{String, String, Int}}) = [(x.name, x.filename, x.lineno) for x in traceback.extract_tb($(e.bref))] ``` - try - for i in 1:nfs - @py ``` - f = $(fs)[$(i-1)] - $(name::String) = f.name - $(fname::String) = f.filename - $(lineno::Int) = f.lineno - ``` - println(io) - printstyled(io, " [", i, "] ") - printstyled(io, name, bold=true) - printstyled(io, " at ") - printstyled(io, fname, ":", lineno, bold=true) - end - finally - C.Py_DecRef(fs) + for (i,(name, fname, lineno)) in enumerate(fs) + println(io) + printstyled(io, " [", i, "] ") + printstyled(io, name, bold=true) + printstyled(io, " at ") + # if (m=match(r"^(.*):([0-9]+)$", fname)) !== nothing + # fname = m.captures[1] + # lineno += parse(Int, m.captures[2]) - 1 + # end + printstyled(io, fname, ":", lineno, bold=true) end - catch - print(io, "") + catch err + print(io, "") end end end diff --git a/src/builtins.jl b/src/builtins.jl index 1716f4cf..9163debb 100644 --- a/src/builtins.jl +++ b/src/builtins.jl @@ -128,7 +128,7 @@ export pyrepr Equivalent to `str(x)` in Python. """ -pystr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Repr, x) +pystr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Str, x) pystr(x) = pystr(PyObject, x) export pystr diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index 49f08c63..17e84b12 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -47,6 +47,7 @@ include("dict.jl") include("set.jl") include("buffer.jl") include("collections.jl") +include("range.jl") __init__() = begin PyObject_TryConvert_AddRules("builtins.NoneType", [ @@ -56,30 +57,36 @@ __init__() = begin PyObject_TryConvert_AddRules("builtins.bool", [ (Bool, PyBool_TryConvertRule_bool, 100), ]) - PyObject_TryConvert_AddRules("builtins.int", [ - (Integer, PyLong_TryConvertRule_integer, 100), - (Rational, PyLong_TryConvertRule_tryconvert), - (Real, PyLong_TryConvertRule_tryconvert), - (Number, PyLong_TryConvertRule_tryconvert), - (Any, PyLong_TryConvertRule_tryconvert), + PyObject_TryConvert_AddRules("numbers.Integral", [ + (Integer, PyLongable_TryConvertRule_integer, 100), + (Rational, PyLongable_TryConvertRule_tryconvert), + (Real, PyLongable_TryConvertRule_tryconvert), + (Number, PyLongable_TryConvertRule_tryconvert), + (Any, PyLongable_TryConvertRule_tryconvert), ]) PyObject_TryConvert_AddRules("builtins.float", [ - (Float64, PyFloat_TryConvertRule_convert, 100), - (BigFloat, PyFloat_TryConvertRule_convert), - (Float32, PyFloat_TryConvertRule_convert), - (Float16, PyFloat_TryConvertRule_convert), - (AbstractFloat, PyFloat_TryConvertRule_tryconvert), - (Real, PyFloat_TryConvertRule_tryconvert), - (Number, PyFloat_TryConvertRule_tryconvert), + (Float64, PyFloatable_TryConvertRule_convert, 100) + ]) + PyObject_TryConvert_AddRules("numbers.Real", [ + (Float64, PyFloatable_TryConvertRule_convert), + (BigFloat, PyFloatable_TryConvertRule_convert), + (Float32, PyFloatable_TryConvertRule_convert), + (Float16, PyFloatable_TryConvertRule_convert), + (AbstractFloat, PyFloatable_TryConvertRule_tryconvert), + (Real, PyFloatable_TryConvertRule_tryconvert), + (Number, PyFloatable_TryConvertRule_tryconvert), ]) PyObject_TryConvert_AddRules("builtins.complex", [ - (Complex{Float64}, PyComplex_TryConvertRule_convert, 100), - (Complex{BigFloat}, PyComplex_TryConvertRule_convert), - (Complex{Float32}, PyComplex_TryConvertRule_convert), - (Complex{Float16}, PyComplex_TryConvertRule_convert), - (Complex{T} where {T<:AbstractFloat}, PyComplex_TryConvertRule_tryconvert), - (Complex{T} where {T<:Real}, PyComplex_TryConvertRule_tryconvert), - (Number, PyComplex_TryConvertRule_tryconvert), + (Complex{Float64}, PyComplexable_TryConvertRule_convert, 100), + ]) + PyObject_TryConvert_AddRules("numbers.Complex", [ + (Complex{Float64}, PyComplexable_TryConvertRule_convert), + (Complex{BigFloat}, PyComplexable_TryConvertRule_convert), + (Complex{Float32}, PyComplexable_TryConvertRule_convert), + (Complex{Float16}, PyComplexable_TryConvertRule_convert), + (Complex{T} where {T<:AbstractFloat}, PyComplexable_TryConvertRule_tryconvert), + (Complex{T} where {T<:Real}, PyComplexable_TryConvertRule_tryconvert), + (Number, PyComplexable_TryConvertRule_tryconvert), ]) PyObject_TryConvert_AddRules("builtins.bytes", [ (Vector{UInt8}, PyBytes_TryConvertRule_vector), @@ -96,6 +103,10 @@ __init__() = begin PyObject_TryConvert_AddRules("builtins.tuple", [ (Tuple, PyIterable_ConvertRule_tuple, 100), ]) + PyObject_TryConvert_AddRules("builtins.range", [ + (StepRange{T,S} where {T<:Integer, S<:Integer}, PyRange_TryConvertRule_steprange, 100), + (UnitRange{T} where {T<:Integer}, PyRange_TryConvertRule_unitrange), + ]) PyObject_TryConvert_AddRules("collections.abc.Iterable", [ (Vector, PyIterable_ConvertRule_vector), (Set, PyIterable_ConvertRule_set), @@ -112,13 +123,37 @@ __init__() = begin (Dict, PyMapping_ConvertRule_dict), ]) PyObject_TryConvert_AddExtraTypes([ - ("collections.abc.Iterable", PyIterableABC_SubclassCheck), - ("collections.abc.Callable", PyCallableABC_SubclassCheck), - ("collections.abc.Sequence", PySequenceABC_SubclassCheck), - ("collections.abc.Mapping", PyMappingABC_SubclassCheck), - ("collections.abc.Set", PySetABC_SubclassCheck), - ("builtins.bufferable", PyType_CheckBuffer), + PyIterableABC_Type, + PyCallableABC_Type, + PySequenceABC_Type, + PyMappingABC_Type, + PySetABC_Type, + PyNumberABC_Type, + PyComplexABC_Type, + PyRealABC_Type, + PyRationalABC_Type, + PyIntegralABC_Type, ]) + ### Numpy + # These aren't necessary but exist just to preserve the datatype. + # TODO: Access these types directly. + # TODO: Compound types? + PyObject_TryConvert_AddRule("numpy.int8", Int8, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.int16", Int16, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.int32", Int32, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.int64", Int64, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.int128", Int128, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.uint8", UInt8, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.uint16", UInt16, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.uint32", UInt32, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.uint64", UInt64, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.uint128", UInt128, PyLongable_TryConvertRule_integer, 100) + PyObject_TryConvert_AddRule("numpy.float16", Float16, PyFloatable_TryConvertRule_convert, 100) + PyObject_TryConvert_AddRule("numpy.float32", Float32, PyFloatable_TryConvertRule_convert, 100) + PyObject_TryConvert_AddRule("numpy.float64", Float64, PyFloatable_TryConvertRule_convert, 100) + PyObject_TryConvert_AddRule("numpy.complex32", Complex{Float16}, PyComplexable_TryConvertRule_convert, 100) + PyObject_TryConvert_AddRule("numpy.complex64", Complex{Float32}, PyComplexable_TryConvertRule_convert, 100) + PyObject_TryConvert_AddRule("numpy.complex128", Complex{Float64}, PyComplexable_TryConvertRule_convert, 100) end end diff --git a/src/cpython/collections.jl b/src/cpython/collections.jl index b7c1586e..521e8c01 100644 --- a/src/cpython/collections.jl +++ b/src/cpython/collections.jl @@ -6,7 +6,6 @@ for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, t = Symbol(p, :_Type) tr = Symbol(p, :__ref) c = Symbol(p, :_Check) - ct = Symbol(p, :_SubclassCheck) @eval const $tr = Ref(PyPtr()) @eval $t() = begin ptr = $tr[] @@ -18,18 +17,11 @@ for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, isnull(b) && return b $tr[] = b end - @eval $c(_o) = GC.@preserve _o begin - o = Base.unsafe_convert(PyPtr, _o) + @eval $c(o) = begin t = $t() isnull(t) && return Cint(-1) PyObject_IsInstance(o, t) end - @eval $ct(_o) = GC.@preserve _o begin - o = Base.unsafe_convert(PyPtr, _o) - t = $t() - isnull(t) && return Cint(-1) - PyObject_IsSubclass(o, t) - end end PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = begin diff --git a/src/cpython/complex.jl b/src/cpython/complex.jl index a35ab534..a002b326 100644 --- a/src/cpython/complex.jl +++ b/src/cpython/complex.jl @@ -1,6 +1,7 @@ @cdef :PyComplex_FromDoubles PyPtr (Cdouble, Cdouble) @cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) @cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) +@cdef :PyComplex_AsCComplex Py_complex (PyPtr,) const PyComplex_Type__ref = Ref(PyPtr()) PyComplex_Type() = pyglobal(PyComplex_Type__ref, :PyComplex_Type) @@ -9,23 +10,23 @@ PyComplex_Check(o) = Py_TypeCheck(o, PyComplex_Type()) PyComplex_CheckExact(o) = Py_TypeCheckExact(o, PyComplex_Type()) +PyComplex_AsComplex(o) = begin + r = PyComplex_AsCComplex(o) + Complex(r.real, r.imag) +end + PyComplex_From(x::Union{Float16,Float32,Float64}) = PyComplex_FromDoubles(x, 0) PyComplex_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_FromDoubles(real(x), imag(x)) -PyComplex_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin - x = PyComplex_RealAsDouble(o) +# "Complexable" means a 'complex' or anything with a '__complex__' method +PyComplexable_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyComplex_AsComplex(o) ism1(x) && PyErr_IsSet() && return -1 - y = PyComplex_ImagAsDouble(o) - ism1(y) && PyErr_IsSet() && return -1 - z = Complex(x, y) - putresult(T, convert(S, z)) + putresult(T, convert(S, x)) end -PyComplex_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin - x = PyComplex_RealAsDouble(o) +PyComplexable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyComplexable_AsComplex(o) ism1(x) && PyErr_IsSet() && return -1 - y = PyComplex_ImagAsDouble(o) - ism1(y) && PyErr_IsSet() && return -1 - z = Complex(x, y) - putresult(T, tryconvert(S, z)) + putresult(T, tryconvert(S, x)) end diff --git a/src/cpython/float.jl b/src/cpython/float.jl index 72c77092..0ecd8b14 100644 --- a/src/cpython/float.jl +++ b/src/cpython/float.jl @@ -10,13 +10,14 @@ PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) -PyFloat_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +# "Floatable" means a 'float' or anything with a '__float__' method +PyFloatable_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin x = PyFloat_AsDouble(o) ism1(x) && PyErr_IsSet() && return -1 putresult(T, convert(S, x)) end -PyFloat_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyFloatable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin x = PyFloat_AsDouble(o) ism1(x) && PyErr_IsSet() && return -1 putresult(T, tryconvert(S, x)) diff --git a/src/cpython/int.jl b/src/cpython/int.jl index 8e25cb8a..b170e267 100644 --- a/src/cpython/int.jl +++ b/src/cpython/int.jl @@ -27,7 +27,8 @@ PyLong_From(x::Integer) = begin PyLong_From(y::BigInt) end -PyLong_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} = begin +# "Longable" means an 'int' or anything with an '__int__' method. +PyLongable_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} = begin # first try to convert to Clonglong (or Culonglong if unsigned) x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) if !ism1(x) || !PyErr_IsSet() @@ -56,7 +57,7 @@ PyLong_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} = b end end -PyLong_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyLongable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin r = PyLong_TryConvertRule_integer(o, Integer, Integer) r == 1 ? putresult(T, tryconvert(S, takeresult(Integer))) : r end diff --git a/src/cpython/number.jl b/src/cpython/number.jl index 6b2937f5..980c9bc1 100644 --- a/src/cpython/number.jl +++ b/src/cpython/number.jl @@ -32,3 +32,26 @@ @cdef :PyNumber_Long PyPtr (PyPtr,) @cdef :PyNumber_Float PyPtr (PyPtr,) @cdef :PyNumber_Index PyPtr (PyPtr,) + +for n in [:Number, :Complex, :Real, :Rational, :Integral] + p = Symbol(:Py, n, :ABC) + t = Symbol(p, :_Type) + tr = Symbol(p, :__ref) + c = Symbol(p, :_Check) + @eval const $tr = Ref(PyPtr()) + @eval $t() = begin + ptr = $tr[] + isnull(ptr) || return ptr + a = PyImport_ImportModule("numbers") + isnull(a) && return a + b = PyObject_GetAttrString(a, $(string(n))) + Py_DecRef(a) + isnull(b) && return b + $tr[] = b + end + @eval $c(o) = begin + t = $t() + isnull(t) && return Cint(-1) + PyObject_IsInstance(o, t) + end +end diff --git a/src/cpython/object.jl b/src/cpython/object.jl index f9290271..58bc68df 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -37,6 +37,7 @@ PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) PyObject_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_From(x) PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) PyObject_From(x::Tuple) = PyTuple_From(x) +PyObject_From(x::AbstractRange{<:Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyRange_From(x) PyObject_From(x::T) where {T} = if ispyreftype(T) GC.@preserve x begin @@ -76,7 +77,7 @@ const ERRPTR = Ptr{Cvoid}(1) const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() const TRYCONVERT_RULES = Dict{String, Vector{Tuple{Int, Type, Function}}}() -const TRYCONVERT_EXTRATYPES = Vector{Tuple{String, Function}}() +const TRYCONVERT_EXTRATYPES = Vector{Function}() @generated PyObject_TryConvert_CompiledRules(::Type{T}) where {T} = get!(Dict{PyPtr, Ptr{Cvoid}}, TRYCONVERT_COMPILED_RULES, T) @@ -88,51 +89,82 @@ PyObject_TryConvert_AddRules(n::String, xs) = for x in xs PyObject_TryConvert_AddRule(n, x...) end -PyObject_TryConvert_AddExtraType(n::String, pred::Function) = - pushfirst!(TRYCONVERT_EXTRATYPES, (n, pred)) +PyObject_TryConvert_AddExtraType(tfunc::Function) = + push!(TRYCONVERT_EXTRATYPES, tfunc) PyObject_TryConvert_AddExtraTypes(xs) = for x in xs - PyObject_TryConvert_AddExtraType(x...) + PyObject_TryConvert_AddExtraType(x) end PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin - # first get the MRO - mrotuple = PyType_MRO(t) - mrotypes = PyPtr[] - mronames = String[] - for i in 1:PyTuple_Size(mrotuple) - b = PyTuple_GetItem(mrotuple, i-1) - isnull(b) && return ERRPTR - push!(mrotypes, b) - n = PyType_FullName(b) - n === PYERR() && return ERRPTR - push!(mronames, n) + + ### STAGE 1: Get a list of supertypes to consider. + # + # This includes the MRO of t, but also the MRO of any extra type (such as an ABC) + # that we explicitly check for because it may not appear the the MRO of t. + + # MRO of t + tmro = PyType_MROAsVector(t) + # find the "MROs" of all base types we are considering + basetypes = PyPtr[t] + basemros = Vector{PyPtr}[tmro] + for xtf in TRYCONVERT_EXTRATYPES + xt = xtf() + isnull(xt) && return ERRPTR + xb = PyPtr() + for b in tmro + r = PyObject_IsSubclass(b, xt) + ism1(r) && return ERRPTR + r != 0 && (xb = b) + end + if xb != PyPtr() + push!(basetypes, xt) + push!(basemros, [xb; PyType_MROAsVector(xt)]) + end end - # find any extra types - extranames = Dict{Int, Vector{String}}() - for (name, pred) in TRYCONVERT_EXTRATYPES - i = nothing - for (j,t) in enumerate(mrotypes) - r = pred(t) - if r === PYERR() - return ERRPTR - elseif r isa Bool - if r - i = j - end - elseif r isa Cint - r == -1 && return ERRPTR - if r != 0 - i = j - end - else - @warn "Unexpected return type from a Python convert extra-types predicate: `$(typeof(r))`" + for b in basetypes[2:end] + push!(basemros, [b]) + end + # merge the MROs + # this is a port of the merge() function at the bottom of: + # https://www.python.org/download/releases/2.3/mro/ + alltypes = PyPtr[] + while !isempty(basemros) + # find the first head not contained in any tail + ok = false + b = PyPtr() + for bmro in basemros + b = bmro[1] + if all(bmro -> b ∉ bmro[2:end], basemros) + ok = true + break end end - i === nothing || push!(get!(Vector{String}, extranames, i), name) + ok || error("Fatal inheritence error: could not merge MROs") + # add it to the list + push!(alltypes, b) + # remove it from consideration + for bmro in basemros + filter!(x -> x != b, bmro) + end + # remove empty lists + filter!(x -> !isempty(x), basemros) end - # merge all the type names together - allnames = [n for (i,mroname) in enumerate(mronames) for n in [mroname; get(Vector{String}, extranames, i)]] + allnames = map(PyType_FullName, alltypes) + # as a special case, we check for the buffer type explicitly + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyType_CheckBuffer(b) + insert!(allnames, i+1, "") + break + end + end + # check the original MRO is preserved + @assert filter(x -> x in tmro, alltypes) == tmro + + ### STAGE 2: Get a list of applicable conversion rules. + # + # These are the conversion rules of the types found above + # gather rules of the form (priority, order, S, rule) from these types rules = Tuple{Int, Int, Type, Function}[] for n in allnames @@ -147,8 +179,11 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin # discard rules where S is a subtype of the union of all previous S for rules with the same implementation # in particular this removes rules with S==Union{} and removes duplicates rules = [S=>rule for (i,(S,rule)) in enumerate(rules) if !(S <: Union{[S′ for (S′,rule′) in rules[1:i-1] if rule==rule′]...})] - # println("CONVERSION RULES FOR '$(PyType_Name(t))' TO '$T':") - # display(rules) + + @debug "PYTHON CONVERSION FOR '$(PyType_FullName(t))' to '$T'" basetypes=map(PyType_FullName, basetypes) alltypes=allnames rules=rules + + ### STAGE 3: Define and compile a function implementing these rules. + # make the function implementing these rules rulefunc = @eval (o::PyPtr) -> begin $((:(r = $rule(o, $T, $S)::Int; r == 0 || return r) for (S,rule) in rules)...) diff --git a/src/cpython/range.jl b/src/cpython/range.jl new file mode 100644 index 00000000..c2686644 --- /dev/null +++ b/src/cpython/range.jl @@ -0,0 +1,90 @@ +const PyRange_Type__ref = Ref(PyPtr()) +PyRange_Type() = begin + ptr = PyRange_Type__ref[] + isnull(ptr) || return ptr + bs = PyEval_GetBuiltins() + ptr = PyMapping_GetItemString(bs, "range") + PyRange_Type__ref[] = ptr +end + +PyRange_From(a::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a) +end + +PyRange_From(a::Integer, b::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a, b) +end + +PyRange_From(a::Integer, b::Integer, c::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a, b, c) +end + +PyRange_From(r::AbstractRange{<:Integer}) = + PyRange_From(first(r), last(r) + sign(step(r)), step(r)) + +steptype(::Type{<:(StepRange{A,B} where {A})}) where {B} = B +steptype(::Type{<:StepRange}) = Any + +PyRange_TryConvertRule_steprange(o, ::Type{T}, ::Type{S}) where {T,S<:StepRange} = begin + A = _typeintersect(Integer, eltype(S)) + B = _typeintersect(Integer, steptype(S)) + # get start + ao = PyObject_GetAttrString(o, "start") + isnull(ao) && return -1 + r = PyObject_TryConvert(ao, A) + Py_DecRef(ao) + r == 1 || return r + a = takeresult(A) + # get step + bo = PyObject_GetAttrString(o, "step") + isnull(bo) && return -1 + r = PyObject_TryConvert(bo, B) + Py_DecRef(bo) + r == 1 || return r + b = takeresult(B) + # get stop + co = PyObject_GetAttrString(o, "stop") + isnull(co) && return -1 + r = PyObject_TryConvert(co, A) + Py_DecRef(co) + r == 1 || return r + c = takeresult(A) + # success + a′, c′ = promote(a, c - oftype(c, sign(b))) + putresult(T, tryconvert(S, StepRange(a′, b, c′))) +end + +PyRange_TryConvertRule_unitrange(o, ::Type{T}, ::Type{S}) where {T,S<:UnitRange} = begin + A = _typeintersect(Integer, eltype(S)) + # get step + bo = PyObject_GetAttrString(o, "step") + isnull(bo) && return -1 + r = PyObject_TryConvert(bo, Int) + Py_DecRef(bo) + r == 1 || return r + b = takeresult(Int) + b == 1 || return 0 + # get start + ao = PyObject_GetAttrString(o, "start") + isnull(ao) && return -1 + r = PyObject_TryConvert(ao, A) + Py_DecRef(ao) + r == 1 || return r + a = takeresult(A) + # get stop + co = PyObject_GetAttrString(o, "stop") + isnull(co) && return -1 + r = PyObject_TryConvert(co, A) + Py_DecRef(co) + r == 1 || return r + c = takeresult(A) + # success + a′, c′ = promote(a, c - oftype(c, 1)) + putresult(T, tryconvert(S, UnitRange(a′, c′))) +end diff --git a/src/cpython/type.jl b/src/cpython/type.jl index 0a723d2d..762c1f71 100644 --- a/src/cpython/type.jl +++ b/src/cpython/type.jl @@ -38,3 +38,8 @@ PyType_FullName(o) = begin # done "$m.$n" end + +PyType_MROAsVector(o) = begin + mro = PyType_MRO(o) + PyPtr[PyTuple_GetItem(mro, i-1) for i in 1:PyTuple_Size(mro)] +end diff --git a/src/eval.jl b/src/eval.jl index ae89228f..e22612e4 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -45,7 +45,7 @@ pyeval_macro(filename, mode, args...) = begin # currently only the pattern "^ $(...) =" is recognized as a LHS # TODO: multiple assignment # TODO: mutating assignment - islhs = [((match(r"\n\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] + islhs = [((match(r"[\n;]\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] # do the interpolation intvars = ["_jl_interp_$(i)_" for i in 1:length(interps)] for (k,v,lhs) in zip(intvars, interps, islhs) diff --git a/src/init.jl b/src/init.jl index a2a781a5..8a4a16f2 100644 --- a/src/init.jl +++ b/src/init.jl @@ -123,6 +123,7 @@ function __init__() end C.PyObject_TryConvert_AddRule("builtins.object", PyObject, CTryConvertRule_wrapref, -100) + C.PyObject_TryConvert_AddRule("builtins.object", PyRef, CTryConvertRule_wrapref, -100) C.PyObject_TryConvert_AddRule("collections.abc.Sequence", PyList, CTryConvertRule_wrapref, 100) C.PyObject_TryConvert_AddRule("collections.abc.Set", PySet, CTryConvertRule_wrapref, 100) C.PyObject_TryConvert_AddRule("collections.abc.Mapping", PyDict, CTryConvertRule_wrapref, 100) From 05dd27d1f936f8b5b5b6f0129321c9d0f5958060 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Fri, 18 Dec 2020 18:52:50 +0000 Subject: [PATCH 09/14] clearing out src/old --- src/PyDict.jl | 2 +- src/PyException.jl | 4 +- src/{old => }/PyIO.jl | 48 +++--- src/PyIterable.jl | 38 +++++ src/PyList.jl | 12 +- src/PyObject.jl | 226 ++++++++++++++++++++++++- src/PyPandasDataFrame.jl | 110 ++++++++++++ src/PyRef.jl | 17 ++ src/PySet.jl | 6 +- src/Python.jl | 79 +-------- src/builtins.jl | 97 +++++++++++ src/cpython/CPython.jl | 82 ++++++--- src/cpython/collections.jl | 8 +- src/cpython/consts.jl | 13 ++ src/cpython/ctypes.jl | 7 + src/cpython/error.jl | 0 src/cpython/fundamentals.jl | 8 + src/cpython/number.jl | 8 +- src/cpython/numpy.jl | 6 + src/cpython/object.jl | 44 ++++- src/eval.jl | 56 +++---- src/gui.jl | 145 ++++++++++++++++ src/old/PyDict.jl | 61 ------- src/old/PyIterable.jl | 31 ---- src/old/base.jl | 325 ------------------------------------ src/old/bool.jl | 22 --- src/old/builtins.jl | 60 ------- src/old/bytearray.jl | 8 - src/old/bytes.jl | 16 -- src/old/collections.jl | 146 ---------------- src/old/complex.jl | 24 --- src/old/convert.jl | 221 ------------------------ src/old/dict.jl | 34 ---- src/old/error.jl | 136 --------------- src/old/eval.jl | 137 --------------- src/old/float.jl | 22 --- src/old/fraction.jl | 33 ---- src/old/gui.jl | 145 ---------------- src/old/import.jl | 34 ---- src/old/int.jl | 67 -------- src/old/io.jl | 5 - src/old/list.jl | 25 --- src/old/none.jl | 12 -- src/old/number.jl | 41 ----- src/old/numpy.jl | 1 - src/old/object.jl | 212 ----------------------- src/old/pandas.jl | 110 ------------ src/old/pywith.jl | 28 ---- src/old/range.jl | 64 ------- src/old/sequence.jl | 2 - src/old/set.jl | 14 -- src/old/slice.jl | 8 - src/old/stdlib.jl | 13 -- src/old/str.jl | 33 ---- src/old/tuple.jl | 28 ---- src/old/type.jl | 12 -- 56 files changed, 838 insertions(+), 2308 deletions(-) rename src/{old => }/PyIO.jl (77%) create mode 100644 src/PyIterable.jl create mode 100644 src/PyPandasDataFrame.jl create mode 100644 src/cpython/ctypes.jl delete mode 100644 src/cpython/error.jl create mode 100644 src/cpython/numpy.jl create mode 100644 src/gui.jl delete mode 100644 src/old/PyDict.jl delete mode 100644 src/old/PyIterable.jl delete mode 100644 src/old/base.jl delete mode 100644 src/old/bool.jl delete mode 100644 src/old/builtins.jl delete mode 100644 src/old/bytearray.jl delete mode 100644 src/old/bytes.jl delete mode 100644 src/old/collections.jl delete mode 100644 src/old/complex.jl delete mode 100644 src/old/convert.jl delete mode 100644 src/old/dict.jl delete mode 100644 src/old/error.jl delete mode 100644 src/old/eval.jl delete mode 100644 src/old/float.jl delete mode 100644 src/old/fraction.jl delete mode 100644 src/old/gui.jl delete mode 100644 src/old/import.jl delete mode 100644 src/old/int.jl delete mode 100644 src/old/io.jl delete mode 100644 src/old/list.jl delete mode 100644 src/old/none.jl delete mode 100644 src/old/number.jl delete mode 100644 src/old/numpy.jl delete mode 100644 src/old/object.jl delete mode 100644 src/old/pandas.jl delete mode 100644 src/old/pywith.jl delete mode 100644 src/old/range.jl delete mode 100644 src/old/sequence.jl delete mode 100644 src/old/set.jl delete mode 100644 src/old/slice.jl delete mode 100644 src/old/stdlib.jl delete mode 100644 src/old/str.jl delete mode 100644 src/old/tuple.jl delete mode 100644 src/old/type.jl diff --git a/src/PyDict.jl b/src/PyDict.jl index 3b12f1cd..d4f7aa96 100644 --- a/src/PyDict.jl +++ b/src/PyDict.jl @@ -72,7 +72,7 @@ Base.length(x::PyDict) = Int(pylen(x)) Base.empty!(x::PyDict) = (@py `$x.clear()`; x) -Base.copy(x::PyDict) = @pyv typeof(x) `$x.copy()` +Base.copy(x::PyDict) = @pyv `$x.copy()`::typeof(x) Base.haskey(x::PyDict{K}, _k) where {K} = begin k = tryconvertref(K, _k) diff --git a/src/PyException.jl b/src/PyException.jl index c1daf413..7d0ffa75 100644 --- a/src/PyException.jl +++ b/src/PyException.jl @@ -115,7 +115,7 @@ function Base.showerror(io::IO, e::PyException) # print the type name try - tname = @pyv String `$(e.tref).__name__` + tname = @pyv `$(e.tref).__name__`::String print(io, tname) catch print(io, "") @@ -125,7 +125,7 @@ function Base.showerror(io::IO, e::PyException) if !isnull(e.vref) print(io, ": ") try - vstr = @pyv String `str($(e.vref))` + vstr = @pyv `str($(e.vref))`::String print(io, vstr) catch print(io, "") diff --git a/src/old/PyIO.jl b/src/PyIO.jl similarity index 77% rename from src/old/PyIO.jl rename to src/PyIO.jl index 7d777139..8e9ea163 100644 --- a/src/old/PyIO.jl +++ b/src/PyIO.jl @@ -10,7 +10,7 @@ If `text=false` then `o` must be a binary stream and arbitrary binary I/O is pos For efficiency, reads and writes are buffered before being sent to `o`. The size of the buffer is `buflen`. """ mutable struct PyIO <: IO - o :: PyObject + ref :: PyRef # true to close the file automatically own :: Bool # true if `o` is text, false if binary @@ -24,8 +24,8 @@ mutable struct PyIO <: IO obuflen :: Int obuf :: Vector{UInt8} - function PyIO(o::PyObject; own::Bool=false, text::Union{Missing,Bool}=missing, buflen::Integer=4096, ibuflen=buflen, obuflen=buflen) - io = new(PyObject(o), own, text===missing ? pyisinstance(o, pyiomodule.TextIOBase) : text, false, ibuflen, UInt8[], obuflen, UInt8[]) + function PyIO(o; own::Bool=false, text::Union{Missing,Bool}=missing, buflen::Integer=4096, ibuflen=buflen, obuflen=buflen) + io = new(PyRef(o), own, text===missing ? pyisinstance(o, pyiomodule().TextIOBase) : text, false, ibuflen, UInt8[], obuflen, UInt8[]) finalizer(io) do io io.own ? close(io) : flush(io) end @@ -34,6 +34,11 @@ mutable struct PyIO <: IO end export PyIO +ispyreftype(::Type{PyIO}) = true +pyptr(io::PyIO) = pyptr(io.ref) +Base.unsafe_convert(::Type{CPyPtr}, io::PyIO) = checknull(pyptr(io)) +C.PyObject_TryConvert__initial(o, ::Type{PyIO}) = C.putresult(PyIO, PyIO(pyborrowedref(o))) + """ PyIO(f, o; ...) @@ -50,12 +55,10 @@ function PyIO(f::Function, o; opts...) end end -pyobject(io::PyIO) = io.o - # If obuf is non-empty, write it to the underlying stream. function putobuf(io::PyIO) if !isempty(io.obuf) - io.text ? io.o.write(String(io.obuf)) : io.o.write(io.obuf) + @py `$io.write($(io.text ? pystr(io.obuf) : pybytes(io.obuf)))` empty!(io.obuf) end nothing @@ -63,26 +66,21 @@ end # If ibuf is empty, read some more from the underlying stream. # After this call, if ibuf is empty then we are at EOF. +# TODO: in binary mode, `io.readinto()` to avoid copying data function getibuf(io::PyIO) if isempty(io.ibuf) - if io.text - append!(io.ibuf, PyVector{UInt8,UInt8,false,true}(pystr_asutf8string(io.o.read(io.ibuflen)))) - else - resize!(io.ibuf, io.ibuflen) - n = io.o.readinto(io.ibuf).jl!i - resize!(io.ibuf, n) - end + append!(io.ibuf, @pyv `$io.read($(io.ibuflen))`::Vector{UInt8}) end nothing end function Base.flush(io::PyIO) putobuf(io) - io.o.flush() + @py `$io.flush()` nothing end -Base.close(io::PyIO) = (flush(io); io.o.close(); nothing) +Base.close(io::PyIO) = (flush(io); @py `$io.close()`; nothing) function Base.eof(io::PyIO) if io.eof @@ -95,13 +93,13 @@ function Base.eof(io::PyIO) end end -Base.fd(io::PyIO) = io.o.fileno().jl!i +Base.fd(io::PyIO) = @pyv `$io.fileno()`::Int -Base.isreadable(io::PyIO) = io.o.readable().jl!b +Base.isreadable(io::PyIO) = @pyv `$io.readable()`::Bool -Base.iswritable(io::PyIO) = io.o.writable().jl!b +Base.iswritable(io::PyIO) = @pyv `$io.writable()`::Bool -Base.isopen(io::PyIO) = !io.o.closed.jl!b +Base.isopen(io::PyIO) = @pyv `not $io.closed`::Bool function Base.unsafe_write(io::PyIO, ptr::Ptr{UInt8}, n::UInt) while true @@ -150,7 +148,7 @@ function Base.seek(io::PyIO, pos::Integer) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(pos, 0) + @py `$io.seek($pos, 0)` io end @@ -164,7 +162,7 @@ function Base.seekstart(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(0, 0) + @py `$io.seek(0, 0)` io end @@ -172,7 +170,7 @@ function Base.seekend(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(0, 2) + @py `$io.seek(0, 2)` io end @@ -188,7 +186,7 @@ function Base.skip(io::PyIO, n::Integer) if 0 ≤ n ≤ io.ibuflen read(io, n) else - io.o.seek(n - length(io.ibuf), 1) + @py `$io.seek($(n - length(io.ibuf)), 1)` empty!(io.ibuf) io.eof = false end @@ -199,11 +197,11 @@ function Base.position(io::PyIO) putobuf(io) if io.text if isempty(io.ibuf) - io.o.position().jl!i + @pyv `$io.position()`::Int else error("`position(io)` text PyIO streams only implemented for empty input buffer (e.g. do `read(io, length(io.ibuf))` first)") end else - io.o.position().jl!i - length(io.ibuf) + (@py `$io.position()`::Int) - length(io.ibuf) end end diff --git a/src/PyIterable.jl b/src/PyIterable.jl new file mode 100644 index 00000000..08f6105e --- /dev/null +++ b/src/PyIterable.jl @@ -0,0 +1,38 @@ +""" + PyIterable{T=PyObject}(o) + +Wrap the Python object `o` into a Julia object which iterates values of type `T`. +""" +struct PyIterable{T} + ref :: PyRef + PyIterable{T}(o) where {T} = new{T}(PyRef(o)) +end +PyIterable(o) = PyIterable{PyObject}(o) +export PyIterable + +ispyreftype(::Type{<:PyIterable}) = true +pyptr(x::PyIterable) = pyptr(x.ref) +Base.unsafe_convert(::Type{CPyPtr}, x::PyIterable) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyIterable} = C.putresult(T, T(pyborrowedref(o))) + +Base.length(x::PyIterable) = Int(pylen(x)) + +Base.IteratorSize(::Type{<:PyIterable}) = Base.SizeUnknown() + +Base.IteratorEltype(::Type{<:PyIterable}) = Base.HasEltype() + +Base.eltype(::Type{PyIterable{T}}) where {T} = T + +function Base.iterate(x::PyIterable{T}, it=pyiter(PyRef, x)) where {T} + vo = C.PyIter_Next(it) + if !isnull(vo) + r = C.PyObject_Convert(vo, T) + C.Py_DecRef(vo) + checkm1(r) + (takeresult(T), it) + elseif C.PyErr_IsSet() + pythrow() + else + nothing + end +end diff --git a/src/PyList.jl b/src/PyList.jl index b89b6c96..8d744292 100644 --- a/src/PyList.jl +++ b/src/PyList.jl @@ -25,15 +25,15 @@ end Base.unsafe_convert(::Type{CPyPtr}, x::PyList) = checknull(pyptr(x)) C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyList} = C.putresult(T, T(pyborrowedref(o))) -# Base.length(x::PyList) = @pyv Int `len($x)` -Base.length(x::PyList) = Int(checkm1(C.PyObject_Length(x))) +# Base.length(x::PyList) = @pyv `len($x)`::Int +Base.length(x::PyList) = Int(pylen(x)) Base.size(x::PyList) = (length(x),) Base.getindex(x::PyList{T}, i::Integer) where {T} = begin checkbounds(x, i) # The following line implements this function, but is typically 3x slower. - # @pyv T `$x[$(i-1)]` + # @pyv `$x[$(i-1)]`::T p = checknull(C.PySequence_GetItem(x, i-1)) r = C.PyObject_Convert(p, T) C.Py_DecRef(p) @@ -59,9 +59,9 @@ Base.push!(x::PyList{T}, v) where {T} = (@py `$x.append($(convertref(T, v)))`; x Base.pushfirst!(x::PyList, v) = insert!(x, 1, v) -Base.pop!(x::PyList{T}) where {T} = @pyv T `$x.pop()` +Base.pop!(x::PyList{T}) where {T} = @pyv `$x.pop()`::T -Base.popat!(x::PyList{T}, i::Integer) where {T} = (checkbounds(x, i); @pyv T `$x.pop($(i-1))`) +Base.popat!(x::PyList{T}, i::Integer) where {T} = (checkbounds(x, i); @pyv `$x.pop($(i-1))`::T) Base.popfirst!(x::PyList) = pop!(x, 1) @@ -72,4 +72,4 @@ Base.sort!(x::PyList; rev::Bool=false) = (@py `$x.sort(reverse=$rev)`; x) Base.empty!(x::PyList) = (@py `$x.clear()`; x) -Base.copy(x::PyList) = @pyv typeof(x) `$x.copy()` +Base.copy(x::PyList) = @pyv `$x.copy()`::typeof(x) diff --git a/src/PyObject.jl b/src/PyObject.jl index 16dad24c..02731fee 100644 --- a/src/PyObject.jl +++ b/src/PyObject.jl @@ -58,9 +58,111 @@ Base.show(io::IO, o::PyObject) = begin end end +function Base.show(io::IO, ::MIME"text/plain", o::PyObject) + h, w = displaysize(io) + h -= 3 + x = try + pystr(String, pypprintmodule().pformat(o, width=w)) + catch + pyrepr(String, o) + end + if get(io, :limit, true) + if '\n' ∈ x + # multiple lines + # each one is truncated to one screen width + print(io, "py:") + h -= 1 + xs = split(x, '\n') + printlines(xs) = + for x in xs + if length(x) ≤ w + print(io, '\n', x) + else + print(io, '\n', x[1:nextind(x, 0, w-1)], '…') + end + end + if length(xs) ≤ h + # all lines fit on screen + printlines(xs) + else + # too many lines, skip the middle ones + h -= 1 + h2 = cld(h, 2) + h3 = (length(xs)+1)-(h-h2) + printlines(xs[1:h2]) + linelen = min( + checkbounds(Bool, xs, h2) ? length(xs[h2]) : 0, + checkbounds(Bool, xs, h3) ? length(xs[h3]) : 0, + w + ) + msg = "... [skipping $(h3-h2-1) lines] ..." + pad = fld(linelen - length(msg), 2) + print(io, "\n", pad > 0 ? " "^pad : "", msg) + printlines(xs[h3:end]) + end + elseif length(x) ≤ w-4 + # one short line + print(io, "py: ", x) + return + else + # one long line + println(io, "py:") + h -= 1 + a = h * w + if length(x) ≤ a + # whole string fits on screen + print(io, x) + else + # too long, skip the middle + h -= 1 + h2 = cld(h, 2) + i2 = nextind(x, 0, h2 * w) + i3 = prevind(x, ncodeunits(x)+1, (h-h2)*w) + println(io, x[1:i2]) + println(io, " ... [skipping $(length(x[nextind(x,i2):prevind(x,i3)])) characters] ...") + print(io, x[i3:end]) + end + end + else + print(io, "py: ", x) + end +end + +Base.show(io::IO, mime::MIME, o::PyObject) = _py_mime_show(io, mime, o) +Base.showable(mime::MIME, o::PyObject) = _py_mime_showable(mime, o) + ### PROPERTIES -Base.getproperty(o::PyObject, k::Symbol) = pygetattr(PyObject, o, k) +Base.getproperty(o::PyObject, k::Symbol) = + if k == :jl! + ((::Type{T}) where {T}) -> pyconvert(T, o) + elseif k == :jl!i + pyconvert(Int, o) + elseif k == :jl!b + pytruth(o) + elseif k == :jl!s + pystr(String, o) + elseif k == :jl!r + pyrepr(String, o) + elseif k == :jl!f + pyconvert(Cdouble, o) + elseif k == :jl!c + pyconvert(Complex{Cdouble}, o) + elseif k == :jl!iter + ((::Type{T}=PyObject) where {T}) -> PyIterable{T}(o) + elseif k == :jl!list + ((::Type{T}=PyObject) where {T}) -> PyList{T}(o) + elseif k == :jl!set + ((::Type{T}=PyObject) where {T}) -> PySet{T}(o) + elseif k == :jl!dict + ((::Type{K}=PyObject, ::Type{V}=PyObject) where {K,V}) -> PyDict{K,V}(o) + elseif k == :jl!io + (; opts...) -> PyIO(o; opts...) + elseif k == :jl!pandasdf + (; opts...) -> PyPandasDataFrame(o; opts...) + else + pygetattr(PyObject, o, k) + end Base.setproperty!(o::PyObject, k::Symbol, v) = pysetattr(o, k, v) @@ -82,9 +184,9 @@ function Base.propertynames(o::PyObject) words.add('__class__') words.update(classmembers(o.__class__)) return words - $words = members($o) + $(words::Vector{Symbol}) = members($o) ``` - [Symbol(pystr(String, x)) for x in words] + words end ### CALL @@ -105,7 +207,7 @@ Base.delete!(o::PyObject, k...) = pydelitem(o, k) Base.length(o::PyObject) = Int(pylen(o)) -Base.iterate(o::PyObject, it::PyRef) = begin +Base.iterate(o::PyObject, it::PyRef=pyiter(PyRef,o)) = begin vo = C.PyIter_Next(it) if !isnull(vo) (pynewobject(vo), it) @@ -115,7 +217,6 @@ Base.iterate(o::PyObject, it::PyRef) = begin nothing end end -Base.iterate(o::PyObject) = iterate(o, pynewref(C.PyObject_GetIter(o), true)) Base.in(x, o::PyObject) = pycontains(o, x) Base.hash(o::PyObject) = trunc(UInt, pyhash(o)) @@ -130,3 +231,118 @@ Base.:(>=)(x::PyObject, y::PyObject) = pyge(PyObject, x, y) Base.:(> )(x::PyObject, y::PyObject) = pygt(PyObject, x, y) Base.isequal(x::PyObject, y::PyObject) = pyeq(Bool, x, y) Base.isless(x::PyObject, y::PyObject) = pylt(Bool, x, y) + +### ARITHMETIC + +Base.zero(::Type{PyObject}) = pyint(0) +Base.one(::Type{PyObject}) = pyint(1) + +# unary +Base.:(-)(o::PyObject) = pyneg(PyObject, o) +Base.:(+)(o::PyObject) = pypos(PyObject, o) +Base.abs(o::PyObject) = pyabs(PyObject, o) +Base.:(~)(o::PyObject) = pyinv(PyObject, o) + +# binary +Base.:(+)(o1::PyObject, o2::PyObject) = pyadd(PyObject, o1, o2) +Base.:(-)(o1::PyObject, o2::PyObject) = pysub(PyObject, o1, o2) +Base.:(*)(o1::PyObject, o2::PyObject) = pymul(PyObject, o1, o2) +Base.:(/)(o1::PyObject, o2::PyObject) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::PyObject, o2::PyObject) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::PyObject, o2::PyObject) = pymod(PyObject, o1, o2) +Base.:(^)(o1::PyObject, o2::PyObject) = pypow(PyObject, o1, o2) +Base.:(<<)(o1::PyObject, o2::PyObject) = pylshift(PyObject, o1, o2) +Base.:(>>)(o1::PyObject, o2::PyObject) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::PyObject, o2::PyObject) = pyand(PyObject, o1, o2) +Base.:xor(o1::PyObject, o2::PyObject) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::PyObject, o2::PyObject) = pyor(PyObject, o1, o2) + +Base.:(+)(o1::PyObject, o2::Number) = pyadd(PyObject, o1, o2) +Base.:(-)(o1::PyObject, o2::Number) = pysub(PyObject, o1, o2) +Base.:(*)(o1::PyObject, o2::Number) = pymul(PyObject, o1, o2) +Base.:(/)(o1::PyObject, o2::Number) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::PyObject, o2::Number) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::PyObject, o2::Number) = pymod(PyObject, o1, o2) +Base.:(^)(o1::PyObject, o2::Number) = pypow(PyObject, o1, o2) +Base.:(<<)(o1::PyObject, o2::Number) = pylshift(PyObject, o1, o2) +Base.:(>>)(o1::PyObject, o2::Number) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::PyObject, o2::Number) = pyand(PyObject, o1, o2) +Base.:xor(o1::PyObject, o2::Number) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::PyObject, o2::Number) = pyor(PyObject, o1, o2) + +Base.:(+)(o1::Number, o2::PyObject) = pyadd(PyObject, o1, o2) # Defining +(::Any, ::PyObject) like this hangs Julia v1.5.2-v1.5.3 (at least) during precompilation +Base.:(-)(o1::Number, o2::PyObject) = pysub(PyObject, o1, o2) +Base.:(*)(o1::Number, o2::PyObject) = pymul(PyObject, o1, o2) +Base.:(/)(o1::Number, o2::PyObject) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::Number, o2::PyObject) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::Number, o2::PyObject) = pymod(PyObject, o1, o2) +# Base.:(^)(o1::Number, o2::PyObject) = pypow(PyObject, o1, o2) +# Base.:(<<)(o1::Number, o2::PyObject) = pylshift(PyObject, o1, o2) +# Base.:(>>)(o1::Number, o2::PyObject) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::Number, o2::PyObject) = pyand(PyObject, o1, o2) +Base.:xor(o1::Number, o2::PyObject) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::Number, o2::PyObject) = pyor(PyObject, o1, o2) + +# ternary +Base.powermod(o1::PyObject, o2::Union{PyObject,Number}, o3::Union{PyObject,Number}) = pypow(PyObject, o1, o2, o3) + +### DOCUMENTATION + +# function Base.Docs.getdoc(o::PyObject) +# function tryget(f, g=identity) +# a = try +# f() +# catch +# return nothing +# end +# pyisnone(a) ? nothing : g(a) +# end +# # function name(o) +# # n = tryget(()->o.__name__) +# # n === nothing && return nothing +# # m = tryget(()->o.__module__) +# # (m === nothing || string(m) == "builtins") ? string(n) : string(m, ".", n) +# # end +# getname(o) = tryget(()->o.__name__, string) +# getdoc(o) = tryget(()->o.__doc__, string ∘ ins.cleandoc) + +# docs = [] + +# # Short description +# ins = pyimport("inspect") +# desc, name = if ins.ismodule(o).jl!b +# (pyhasattr(o, "__path__") ? "package" : "module"), gname(o) +# elseif ins.isgetsetdescriptor(o).jl!b +# ("getset descriptor", "$(getname(o.__objclass__)).$(o.__name__)") +# elseif ins.ismemberdescriptor(o).jl!b +# ("member descriptor", "$(getname(o.__objclass__)).$(o.__name__)") +# elseif ins.isclass(o).jl!b +# ("class", getname(o)) +# elseif ins.isfunction(o).jl!b || ins.isbuiltin(o).jl!b +# ("function", getname(o)) +# elseif ins.ismethod(o).jl!b +# ("method", getname(o)) +# else +# ("object of type", getname(pytype(o))) +# end +# push!(docs, Markdown.Paragraph(["Python ", desc, " ", Markdown.Code(name), "."])) + +# if ins.isroutine(o).jl!b || ins.isclass(o).jl!b +# try +# push!(docs, Markdown.Code("python", "$(o.__name__)$(ins.signature(o))")) +# catch +# end +# end + +# # Maybe document the class instead +# doc = getdoc(o) +# if doc === nothing && !ins.ismodule(o).jl!b && !ins.isclass(o).jl!b && !ins.isroutine(o).jl!b && !ins.isdatadescriptor(o).jl!b +# o = pyhasattr(o, "__origin__") ? o.__origin__ : pytype(o) +# doc = getdoc(o) +# end +# doc === nothing || push!(docs, Markdown.Paragraph([Markdown.Text(doc)])) + +# # Done +# Markdown.MD(docs) +# end +# Base.Docs.Binding(o::PyObject, k::Symbol) = getproperty(o, k) diff --git a/src/PyPandasDataFrame.jl b/src/PyPandasDataFrame.jl new file mode 100644 index 00000000..3ce5306a --- /dev/null +++ b/src/PyPandasDataFrame.jl @@ -0,0 +1,110 @@ +# const pypandas = pylazyobject(() -> pyimport("pandas")) +# const pypandasdataframetype = pylazyobject(() -> pypandas.DataFrame) + +# asvector(x::AbstractVector) = x +# asvector(x) = collect(x) + +# """ +# pycolumntable(src) + +# Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. +# """ +# function pycolumntable(src) +# cols = Tables.columns(src) +# pydict_fromiter(pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) +# end +# pycolumntable(; cols...) = pycolumntable(cols) +# export pycolumntable + +# """ +# pyrowtable(src) + +# Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. +# """ +# function pyrowtable(src) +# rows = Tables.rows(src) +# names = Tables.columnnames(rows) +# pynames = [pystr(String(n)) for n in names] +# pylist_fromiter(pydict_fromiter(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) +# end +# pyrowtable(; cols...) = pyrowtable(cols) +# export pyrowtable + +# """ +# pypandasdataframe([src]; ...) + +# Construct a pandas dataframe from `src`. + +# Usually equivalent to `pyimport("pandas").DataFrame(src, ...)`, but `src` may also be `Tables.jl`-compatible table. +# """ +# pypandasdataframe(t::PyObject; opts...) = pypandasdataframetype(t; opts...) +# pypandasdataframe(; opts...) = pypandasdataframetype(; opts...) +# function pypandasdataframe(t; opts...) +# if Tables.istable(t) +# cs = Tables.columns(t) +# pypandasdataframetype(pydict_fromstringiter(string(c) => asvector(Tables.getcolumn(cs, c)) for c in Tables.columnnames(cs)); opts...) +# else +# pypandasdataframetype(t; opts...) +# end +# end +# export pypandasdataframe + +multidict(src) = Dict(k=>v for (ks,v) in src for k in (ks isa Vector ? ks : [ks])) + +""" + PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) + +Wrap the Pandas dataframe `o` as a Julia table. + +This object satisfies the `Tables.jl` and `TableTraits.jl` interfaces. + +- `:indexname` is the name of the index column when converting this to a table, and may be `nothing` to exclude the index. +- `:columntypes` is an iterable of `columnname=>type` or `[columnnames...]=>type` pairs, used when converting to a table. +- `:copy` is true to copy columns on conversion. +""" +struct PyPandasDataFrame + ref :: PyRef + indexname :: Union{Symbol, Nothing} + columntypes :: Dict{Symbol, Type} + copy :: Bool +end +PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) = PyPandasDataFrame(PyRef(o), indexname, multidict(columntypes), copy) +export PyPandasDataFrame + +ispyreftype(::Type{PyPandasDataFrame}) = true +pyptr(df::PyPandasDataFrame) = df.ref +Base.unsafe_convert(::Type{CPyPtr}, df::PyPandasDataFrame) = checknull(pyptr(df)) +C.PyObject_TryConvert__initial(o, ::Type{PyPandasDataFrame}) = C.putresult(PyPandasDataFrame, PyPandasDataFrame(pyborrowedref(o))) + +Base.show(io::IO, x::PyPandasDataFrame) = print(io, pystr(String, x)) +Base.show(io::IO, mime::MIME, o::PyPandasDataFrame) = _py_mime_show(io, mime, o) +Base.showable(mime::MIME, o::PyPandasDataFrame) = _py_mime_showable(mime, o) + +### Tables.jl / TableTraits.jl integration + +Tables.istable(::Type{PyPandasDataFrame}) = true +Tables.columnaccess(::Type{PyPandasDataFrame}) = true +function Tables.columns(x::PyPandasDataFrame) + error("not implemented") + # # columns + # names = Symbol[Symbol(pystr(String, c)) for c in x.o.columns] + # columns = PyVector[PyVector{get(x.columntypes, c, missing)}(x.o[pystr(c)].to_numpy()) for c in names] + # # index + # if x.indexname !== nothing + # if x.indexname ∈ names + # error("table already has column called $(x.indexname), cannot use it for index") + # else + # pushfirst!(names, x.indexname) + # pushfirst!(columns, PyArray{get(x.columntypes, x.indexname, missing)}(x.o.index.to_numpy())) + # end + # end + # if x.copy + # columns = map(copy, columns) + # end + # return NamedTuple{Tuple(names)}(Tuple(columns)) +end + +IteratorInterfaceExtensions.isiterable(x::PyPandasDataFrame) = true +IteratorInterfaceExtensions.getiterator(x::PyPandasDataFrame) = IteratorInterfaceExtensions.getiterator(Tables.rows(x)) + +TableTraits.isiterabletable(x::PyPandasDataFrame) = true diff --git a/src/PyRef.jl b/src/PyRef.jl index 8ad6bc2e..2bcddec4 100644 --- a/src/PyRef.jl +++ b/src/PyRef.jl @@ -47,3 +47,20 @@ PyRef() = pynewref(CPyPtr()) Base.convert(::Type{PyRef}, x::PyRef) = x Base.convert(::Type{PyRef}, x) = PyRef(x) + +# Cache some common standard modules +for name in ["os", "io", "sys", "pprint", "collections", "collections.abc", "numbers", "fractions", "datetime", "numpy", "pandas"] + f = Symbol("py", replace(name, "."=>""), "module") + rf = Symbol("_", f) + @eval $rf = PyRef() + @eval $f(::Type{T}) where {T} = begin + r = $rf + if isnull(r.ptr) + m = pyimport(PyRef, $name) + C.Py_IncRef(m.ptr) + r.ptr = m.ptr + end + (r isa T) ? r : pyconvert(T, r) + end + @eval $f() = $f(PyObject) +end diff --git a/src/PySet.jl b/src/PySet.jl index 8b2c416f..9308ea0e 100644 --- a/src/PySet.jl +++ b/src/PySet.jl @@ -44,7 +44,7 @@ Base.iterate(x::PySet) = begin iterate(x, pynewref(it)) end -# Base.length(x::PySet) = @pyv Int `len($x)` +# Base.length(x::PySet) = @pyv `len($x)`::Int Base.length(x::PySet) = Int(checkm1(C.PyObject_Length(x))) Base.in(_v, x::PySet{T}) where {T} = begin @@ -64,7 +64,7 @@ Base.delete!(x::PySet{T}, _v) where {T} = begin x end -Base.pop!(x::PySet{T}) where {T} = @pyv T `$x.pop()` +Base.pop!(x::PySet{T}) where {T} = @pyv `$x.pop()`::T Base.pop!(x::PySet{T}, _v) where {T} = begin v = tryconvert(T, _v) @@ -80,4 +80,4 @@ end Base.empty!(x::PySet) = (@py `$x.clear()`; x) -Base.copy(x::PySet) = @pyv typeof(x) `$x.copy()` +Base.copy(x::PySet) = @pyv `$x.copy()`::typeof(x) diff --git a/src/Python.jl b/src/Python.jl index c3d7d0af..e108064b 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -78,14 +78,7 @@ include("cpython/CPython.jl") const C = CPython const CPyPtr = C.PyPtr -# struct CPyObjRef -# ptr :: CPyPtr -# end - -# # core -# include("object.jl") -# include("error.jl") -# include("import.jl") + include("gil.jl") include("eval.jl") include("builtins.jl") @@ -98,72 +91,12 @@ include("PyObject.jl") include("PyDict.jl") include("PyList.jl") include("PySet.jl") +include("PyIterable.jl") +include("PyIO.jl") +include("PyPandasDataFrame.jl") + +include("gui.jl") -# # abstract interfaces -# include("number.jl") -# include("sequence.jl") - -# # fundamental objects -# include("type.jl") -# include("none.jl") - -# # numeric objects -# include("bool.jl") -# include("int.jl") -# include("float.jl") -# include("complex.jl") - -# # sequence objects -# include("str.jl") -# include("bytes.jl") -# include("bytearray.jl") -# include("tuple.jl") -# include("list.jl") - -# # mapping objects -# include("dict.jl") -# include("set.jl") - -# # function objects -# include("function.jl") - -# # other objects -# include("slice.jl") -# include("range.jl") - -# # standard library -# include("builtins.jl") -# include("eval.jl") -# include("stdlib.jl") -# include("io.jl") -# include("fraction.jl") -# include("datetime.jl") -# include("collections.jl") - -# # other packages -# include("pandas.jl") -# include("numpy.jl") -# include("matplotlib.jl") - -# # other Julia wrappers around Python values -# include("PyIterable.jl") -# include("PyList.jl") -# include("PyDict.jl") -# include("PySet.jl") -# include("PyObjectArray.jl") -# include("PyBuffer.jl") -# include("PyArray.jl") -# include("PyIO.jl") - -# # other functionality -# include("convert.jl") -# include("newtype.jl") -# include("julia.jl") -# include("base.jl") -# include("pywith.jl") -# include("gui.jl") - -# initialize include("init.jl") end # module diff --git a/src/builtins.jl b/src/builtins.jl index 9163debb..8bb457ff 100644 --- a/src/builtins.jl +++ b/src/builtins.jl @@ -39,6 +39,7 @@ cpyop(::Type{T}, f::Function, args...) where {T} = checknullconvert(T, cpyop(f, Convert Python object `x` to a `T`. """ pyconvert(::Type{T}, x) where {T} = checknullconvert(T, C.PyObject_From(x)) +export pyconvert """ pyis(x, y) :: Bool @@ -64,6 +65,7 @@ pyis(x::X, y::Y) where {X,Y} = begin end xo == yo end +export pyis """ pyhasattr(x, k) :: Bool @@ -129,9 +131,20 @@ export pyrepr Equivalent to `str(x)` in Python. """ pystr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Str, x) +pystr(::Type{T}, x::Union{String, SubString{String}, Vector{Int8}, Vector{UInt8}}) where {T} = checknullconvert(T, C.PyUnicode_From(x)) pystr(x) = pystr(PyObject, x) export pystr +""" + pybytes([T=PyObject,] x) :: T + +Equivalent to `str(x)` in Python. +""" +pybytes(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Bytes, x) +pybytes(::Type{T}, x::Union{Vector{Int8}, Vector{UInt8}, String, SubString{String}}) where {T} = checknullconvert(T, C.PyBytes_From(x)) +pybytes(x) = pybytes(PyObject, x) +export pybytes + """ pylen(x) :: Integer @@ -626,3 +639,87 @@ Equivalent to `-x` in Python. pyinv(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Invert, x) pyinv(x) = pyinv(typeof(x), x) export pyinv + +# pycollist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pycollist(y) for y in eachslice(x; dims=N)) +# export pycollist + +# pyrowlist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pyrowlist(y) for y in eachslice(x; dims=1)) +# export pyrowlist + +""" + pyiter([T=PyObject] x) :: T + +Equivalent to `iter(x)` in Python. +""" +pyiter(::Type{T}, x) where {T} = cpyop(T, C.PyObject_GetIter, x) +pyiter(x) = pyiter(PyObject, x) +export pyiter + +""" + pywith(f, o, d=nothing) + +Equivalent to `with o as x: f(x)` in Python, where `x` is a `PyObject`. + +On success, the value of `f(x)` is returned. +If an exception occurs but is suppressed then `d` is returned. +""" +function pywith(f, _o, d=nothing) + o = PyObject(_o) + t = pytype(o) + exit = t.__exit__ + value = t.__enter__(o) + exited = false + try + return f(value) + catch err + if err isa PyException + exited = true + if pytruth(exit(o, err.tref, err.vref, err.bref)) + rethrow() + else + return d + end + else + rethrow() + end + finally + exited || exit(o, pynone(), pynone(), pynone()) + end +end +export pywith + +### MULTIMEDIA DISPLAY + +for (mime, method) in [ + (MIME"text/html", "_repr_html_"), (MIME"text/markdown", "_repr_markdown_"), + (MIME"text/json", "_repr_json_"), (MIME"application/javascript", "_repr_javascript_"), + (MIME"application/pdf", "_repr_pdf_"), (MIME"image/jpeg", "_repr_jpeg_"), + (MIME"image/png", "_repr_png_"), (MIME"image/svg+xml", "_repr_svg_"), + (MIME"text/latex", "_repr_latex_") + ] + + T = istextmime(mime()) ? String : Vector{UInt8} + @eval begin + _py_mime_show(io::IO, mime::$mime, o) = begin + try + x = pycall(PyRef, pygetattr(Ref, o, $method)) + pyis(x, pynone(PyRef)) || return write(io, pyconvert($T, x)) + catch + end + throw(MethodError(show, (io, mime, o))) + end + _py_mime_showable(::$mime, o) = begin + try + x = pycall(PyRef, pygetattr(PyRef, o, $method)) + if pyis(x, pynone(PyRef)) + false + else + pyconvert($T, x) + true + end + catch + false + end + end + end +end diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index 17e84b12..06eb03e4 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -48,6 +48,8 @@ include("set.jl") include("buffer.jl") include("collections.jl") include("range.jl") +include("ctypes.jl") +include("numpy.jl") __init__() = begin PyObject_TryConvert_AddRules("builtins.NoneType", [ @@ -134,26 +136,66 @@ __init__() = begin PyRationalABC_Type, PyIntegralABC_Type, ]) - ### Numpy - # These aren't necessary but exist just to preserve the datatype. - # TODO: Access these types directly. - # TODO: Compound types? - PyObject_TryConvert_AddRule("numpy.int8", Int8, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.int16", Int16, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.int32", Int32, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.int64", Int64, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.int128", Int128, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.uint8", UInt8, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.uint16", UInt16, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.uint32", UInt32, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.uint64", UInt64, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.uint128", UInt128, PyLongable_TryConvertRule_integer, 100) - PyObject_TryConvert_AddRule("numpy.float16", Float16, PyFloatable_TryConvertRule_convert, 100) - PyObject_TryConvert_AddRule("numpy.float32", Float32, PyFloatable_TryConvertRule_convert, 100) - PyObject_TryConvert_AddRule("numpy.float64", Float64, PyFloatable_TryConvertRule_convert, 100) - PyObject_TryConvert_AddRule("numpy.complex32", Complex{Float16}, PyComplexable_TryConvertRule_convert, 100) - PyObject_TryConvert_AddRule("numpy.complex64", Complex{Float32}, PyComplexable_TryConvertRule_convert, 100) - PyObject_TryConvert_AddRule("numpy.complex128", Complex{Float64}, PyComplexable_TryConvertRule_convert, 100) + + ### ctypes + for (p,T) in [("char", Cchar), ("wchar", Cwchar_t), ("byte", Cchar), ("ubyte", Cuchar), + ("short", Cshort), ("ushort", Cushort), ("int", Cint), ("uint", Cuint), + ("long", Clong), ("ulong", Culong), ("longlong", Culonglong), ("size_t", Csize_t), + ("ssize_t", Cssize_t), ("float", Cfloat), ("double", Cdouble), #=("longdouble", ???),=# + ("char_p", Ptr{Cchar}), ("wchar_p", Ptr{Cwchar_t}), ("void_p", Ptr{Cvoid})] + isptr = occursin("_p", p) + isfloat = occursin("float", p) || occursin("double", p) + isint = !(isfloat || isptr) + isreal = isint || isfloat + PyObject_TryConvert_AddRules("ctypes.c_$p", [ + (p=="char_p" ? Cstring : p=="wchar_p" ? Cwstring : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (T, PySimpleCData_TryConvert_value{T,false}()), + (isint ? Integer : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isint ? Rational : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Float64 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? BigFloat : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? Float32 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? Float16 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? AbstractFloat : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Real : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Number : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isptr ? Ptr : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (Any, PySimpleCData_TryConvert_value{T,true}()), + ]) + end + + ### numpy + # TODO: Compound types + # TODO: datetime64, timedelta64 + for (p,T) in [("int8", Int8), ("int16", Int16), ("int32", Int32), ("int64", Int64), + ("int128", Int128), ("uint8", UInt8), ("uint16", UInt16), ("uint32", UInt32), + ("uint64", UInt64), ("uint128", UInt128), ("float16", Float16), ("float32", Float32), + ("float64", Float64), ("complex32", Complex{Float16}), + ("complex64", Complex{Float32}), ("complex128", Complex{Float64})] + isint = occursin("int", p) + isfloat = occursin("float", p) + iscomplex = occursin("complex", p) + isreal = isint || isfloat + PyObject_TryConvert_AddRules("numpy.$p", [ + (T, PyNumpySimpleData_TryConvert_value{T,false}(), 100), + (isint ? Integer : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isint ? Rational : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isreal ? Float64 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? BigFloat : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? Float32 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? Float16 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? AbstractFloat : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isreal ? Real : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (iscomplex ? Complex{Float64} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{BigFloat} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{Float32} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{Float16} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? (Complex{T} where {T<:AbstractFloat}) : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (iscomplex ? (Complex{T} where {T<:Real}) : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (Number, PyNumpySimpleData_TryConvert_value{T,true}()), + (Any, PyNumpySimpleData_TryConvert_value{T,true}()), + ]) + end end end diff --git a/src/cpython/collections.jl b/src/cpython/collections.jl index 521e8c01..8045c729 100644 --- a/src/cpython/collections.jl +++ b/src/cpython/collections.jl @@ -7,10 +7,10 @@ for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, tr = Symbol(p, :__ref) c = Symbol(p, :_Check) @eval const $tr = Ref(PyPtr()) - @eval $t() = begin + @eval $t(doimport::Bool=true) = begin ptr = $tr[] isnull(ptr) || return ptr - a = PyImport_ImportModule("collections.abc") + a = doimport ? PyImport_ImportModule("collections.abc") : PyImport_GetModule("collections.abc") isnull(a) && return a b = PyObject_GetAttrString(a, $(string(n))) Py_DecRef(a) @@ -18,8 +18,8 @@ for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, $tr[] = b end @eval $c(o) = begin - t = $t() - isnull(t) && return Cint(-1) + t = $t(false) + isnull(t) && return (PyErr_IsSet() ? Cint(-1) : Cint(0)) PyObject_IsInstance(o, t) end end diff --git a/src/cpython/consts.jl b/src/cpython/consts.jl index 36cc2334..6401c3c0 100644 --- a/src/cpython/consts.jl +++ b/src/cpython/consts.jl @@ -307,3 +307,16 @@ end end const PyTypePtr = Ptr{PyTypeObject} + +@kwdef struct PySimpleObject{T} + ob_base :: PyObject = PyObject() + value :: T +end + +PySimpleObject_GetValue(__o, ::Type{T}) where {T} = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + UnsafePtr{PySimpleObject{T}}(o).value[!] :: T + end +end diff --git a/src/cpython/ctypes.jl b/src/cpython/ctypes.jl new file mode 100644 index 00000000..258276d5 --- /dev/null +++ b/src/cpython/ctypes.jl @@ -0,0 +1,7 @@ +struct PySimpleCData_TryConvert_value{R,tr} end + +(::PySimpleCData_TryConvert_value{R,tr})(o, ::Type{T}, ::Type{S}) where {T,S,R,tr} = begin + ptr = PySimpleObject_GetValue(o, Ptr{R}) + val = unsafe_load(ptr) + putresult(T, tr ? tryconvert(S, val) : convert(S, val)) +end diff --git a/src/cpython/error.jl b/src/cpython/error.jl deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cpython/fundamentals.jl b/src/cpython/fundamentals.jl index fa873c68..7c48d29b 100644 --- a/src/cpython/fundamentals.jl +++ b/src/cpython/fundamentals.jl @@ -39,6 +39,14 @@ Py_Is(o1, o2) = Base.unsafe_convert(PyPtr, o1) == Base.unsafe_convert(PyPtr, o2) @cdef :PyImport_ImportModule PyPtr (Cstring,) @cdef :PyImport_Import PyPtr (PyPtr,) +@cdef :PyImport_GetModuleDict PyPtr () + +PyImport_GetModule(name) = begin + ms = PyImport_GetModuleDict() + ok = PyMapping_HasKeyString(ms, name) + ism1(ok) && return PyPtr() + ok != 0 ? PyMapping_GetItemString(ms, name) : PyPtr() +end ### ERRORS diff --git a/src/cpython/number.jl b/src/cpython/number.jl index 980c9bc1..7fa2ac1e 100644 --- a/src/cpython/number.jl +++ b/src/cpython/number.jl @@ -39,10 +39,10 @@ for n in [:Number, :Complex, :Real, :Rational, :Integral] tr = Symbol(p, :__ref) c = Symbol(p, :_Check) @eval const $tr = Ref(PyPtr()) - @eval $t() = begin + @eval $t(doimport::Bool=true) = begin ptr = $tr[] isnull(ptr) || return ptr - a = PyImport_ImportModule("numbers") + a = doimport ? PyImport_ImportModule("numbers") : PyImport_GetModule("numbers") isnull(a) && return a b = PyObject_GetAttrString(a, $(string(n))) Py_DecRef(a) @@ -50,8 +50,8 @@ for n in [:Number, :Complex, :Real, :Rational, :Integral] $tr[] = b end @eval $c(o) = begin - t = $t() - isnull(t) && return Cint(-1) + t = $t(false) + isnull(t) && return (PyErr_IsSet() ? Cint(-1) : Cint(0)) PyObject_IsInstance(o, t) end end diff --git a/src/cpython/numpy.jl b/src/cpython/numpy.jl new file mode 100644 index 00000000..dc74ee04 --- /dev/null +++ b/src/cpython/numpy.jl @@ -0,0 +1,6 @@ +struct PyNumpySimpleData_TryConvert_value{R,tr} end + +(::PyNumpySimpleData_TryConvert_value{R,tr})(o, ::Type{T}, ::Type{S}) where {T,S,R,tr} = begin + val = PySimpleObject_GetValue(o, R) + putresult(T, tr ? tryconvert(S, val) : convert(S, val)) +end diff --git a/src/cpython/object.jl b/src/cpython/object.jl index 58bc68df..d35161a9 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -75,21 +75,53 @@ PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) const ERRPTR = Ptr{Cvoid}(1) +""" +Mapping of Julia types to mappings of Python types to compiled functions implementing the conversion. + +That is, `ccall(TRYCONVERT_COMPILED_RULES[T][Py_Type(o)], Int, (PyPtr,), o)` attempts to convert `o` to a `T`. +On success, this returns `1` and the result can be obtained with `takeresult(T)`. +Otherwise returns `0` if no conversion was possible, or `-1` on error. +""" const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() -const TRYCONVERT_RULES = Dict{String, Vector{Tuple{Int, Type, Function}}}() -const TRYCONVERT_EXTRATYPES = Vector{Function}() + +""" +Mapping of type names to lists of rules. + +A rule is a triple `(priority, type, impl)`. + +Rules are applied in priority order, then in MRO order, then in the order in this list. You should currently only use the following values: +- `0` (the default) for most rules. +- `100` for "canonical" rules which are the preferred way to convert a type, e.g. `float` to `Float64`. +- `-100` for "last ditch" rules once all other options have been exhausted (currently used for anything to `PyObject` or `PyRef`). + +The rule is applied only when the `type` has non-trivial intersection with the target type `T`. + +Finally `impl` is the function implementing the conversion. +Its signature is `(o, T, S)` where `o` is the `PyPtr` object to convert, `T` is the desired type, and `S = typeintersect(T, type)`. +On success it returns `putresult(T, x)` where `x` is the converted value (this stores the result and returns `1`). +If conversion was not possible, returns `0` (indicating we move on to the next rule in the list). +On error, returns `-1`. +""" +const TRYCONVERT_RULES = Dict{String, Vector{Tuple{Int, Type, Any}}}() + +""" +List of niladic functions returning a pointer to a type. We always check for subtypes of these types in TryConvert. + +Can also return NULL. Without an error set, this indicates the type is not loaded (e.g. its containing module is not loaded) and therefore is skipped over. +""" +const TRYCONVERT_EXTRATYPES = Vector{Any}() @generated PyObject_TryConvert_CompiledRules(::Type{T}) where {T} = get!(Dict{PyPtr, Ptr{Cvoid}}, TRYCONVERT_COMPILED_RULES, T) PyObject_TryConvert_Rules(n::String) = get!(Vector{Tuple{Int, Type, Function}}, TRYCONVERT_RULES, n) -PyObject_TryConvert_AddRule(n::String, T::Type, rule::Function, priority::Int=0) = +PyObject_TryConvert_AddRule(n::String, T, rule, priority=0) = push!(PyObject_TryConvert_Rules(n), (priority, T, rule)) PyObject_TryConvert_AddRules(n::String, xs) = for x in xs PyObject_TryConvert_AddRule(n, x...) end -PyObject_TryConvert_AddExtraType(tfunc::Function) = +PyObject_TryConvert_AddExtraType(tfunc) = push!(TRYCONVERT_EXTRATYPES, tfunc) PyObject_TryConvert_AddExtraTypes(xs) = for x in xs @@ -110,7 +142,7 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin basemros = Vector{PyPtr}[tmro] for xtf in TRYCONVERT_EXTRATYPES xt = xtf() - isnull(xt) && return ERRPTR + isnull(xt) && (PyErr_IsSet() ? (return ERRPTR) : continue) xb = PyPtr() for b in tmro r = PyObject_IsSubclass(b, xt) @@ -166,7 +198,7 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin # These are the conversion rules of the types found above # gather rules of the form (priority, order, S, rule) from these types - rules = Tuple{Int, Int, Type, Function}[] + rules = Tuple{Int, Int, Type, Any}[] for n in allnames for (p, S, r) in PyObject_TryConvert_Rules(n) push!(rules, (p, length(rules)+1, S, r)) diff --git a/src/eval.jl b/src/eval.jl index e22612e4..2171f93f 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,18 +1,19 @@ pyeval_filename(src) = isfile(string(src.file)) ? "$(src.file):$(src.line)" : "julia:$(src.file):$(src.line)" -pyeval_macro(filename, mode, args...) = begin - # find the code argument - icode = findfirst(args) do x - x isa Expr && x.head == :macrocall && x.args[1] == :(`foo`).args[1] +pyeval_macro(filename, mode, codearg, args...) = begin + # unwrap rettype + if codearg isa Expr && codearg.head == :(::) + rettypearg = codearg.args[end] + codearg = codearg.args[1] + else + rettypearg = mode == :eval ? PyObject : Nothing end - icode in (1,2) || error() - code = args[icode].args[end] - # the return type - rettypearg = icode==2 ? args[1] : mode==:eval ? PyObject : Nothing + # extract the code + code = codearg.args[end] # parse the remaining arguments - the first one is optionally the locals, the rest are var=value pairs to include in the locals kvs = [] locals = nothing - for (i,arg) in enumerate(args[icode+1:end]) + for (i,arg) in enumerate(args) if arg isa Expr && arg.head == :(=) && arg.args[1] isa Symbol push!(kvs, arg.args[1] => arg.args[2]) elseif i == 1 @@ -45,7 +46,7 @@ pyeval_macro(filename, mode, args...) = begin # currently only the pattern "^ $(...) =" is recognized as a LHS # TODO: multiple assignment # TODO: mutating assignment - islhs = [((match(r"[\n;]\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] + islhs = [((match(r"[\n;=]\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] # do the interpolation intvars = ["_jl_interp_$(i)_" for i in 1:length(interps)] for (k,v,lhs) in zip(intvars, interps, islhs) @@ -134,10 +135,14 @@ end Executes the given Python code. Julia values can be interpolated using the usual `\$(...)` syntax. + Additionally, assignment to interpolations is supported: e.g. `\$(x::T) = ...` will convert the right hand side to a `T` and assign it to `x`. -Currently only single assignment is supported, and it must occur at the start of a line; multiple assignment (`\$x, \$y = ...`) or mutating assignment (`\$x += ...`) will not be recognized. +- Currently only single assignment is supported. Multiple assignment (`\$x, \$y = ...`) or mutating assignment (`\$x += ...`) will not be recognized. +- What actually happens is that the assignment is to a temporary Python variable, which is then read when execution successfully finishes. + Hence if an exception occurs, no assignments will happen. -The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. +The globals are `pyglobals`. +The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. """ macro py(args...) pyeval_macro(pyeval_filename(__source__), :exec, args...) @@ -145,23 +150,26 @@ end export @py """ - py"..." + @pyg `...` [var=val, ...] + +Executes the given Python code in the global scope. -Shorthand for ```@py `...` ```. +This is simply shorthand for ```@py `...` pyglobals ``` (see [`@py`](@ref)). """ -macro py_str(code::String) - pyeval_macro(pyeval_filename(__source__), :exec, Expr(:macrocall, :(`foo`).args[1], code)) +macro pyg(code, args...) + :(@py $code $(esc(:pyglobals)) $(args...)) end -export @py_str +export @pyg """ - @pyv [rettype] `...` [locals] [var=val, ...] + @pyv `...`[::rettype] [locals] [var=val, ...] Evaluate the given Python code. Julia values can be interpolated using the usual `\$(...)` syntax. -The globals are `pyglobals`. The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. +The globals are `pyglobals`. +The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. The result is converted to a `rettype`, which defaults to `PyObject`. """ @@ -170,16 +178,6 @@ macro pyv(args...) end export @pyv -""" - pyv"..." - -Shorthand for ```@pyv `...` ```. -""" -macro pyv_str(code::String) - pyeval_macro(pyeval_filename(__source__), :eval, Expr(:macrocall, :(`foo`).args[1], code)) -end -export @pyv_str - """ py`...` :: PyCode diff --git a/src/gui.jl b/src/gui.jl new file mode 100644 index 00000000..459d70ba --- /dev/null +++ b/src/gui.jl @@ -0,0 +1,145 @@ +""" + fix_qt_plugin_path() + +Try to set the `QT_PLUGIN_PATH` environment variable in Python, if not already set. + +This fixes the problem that Qt does not know where to find its `qt.conf` file, because it +always looks relative to `sys.executable`, which can be the Julia executable not the Python +one when using this package. + +If `CONFIG.qtfix` is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide` or `PySide2` are imported. +""" +function fix_qt_plugin_path() + CONFIG.exepath === nothing && return false + e = pyosmodule().environ + "QT_PLUGIN_PATH" in e && return false + qtconf = joinpath(dirname(CONFIG.exepath), "qt.conf") + isfile(qtconf) || return false + for line in eachline(qtconf) + m = match(r"^\s*prefix\s*=(.*)$"i, line) + if m !== nothing + path = strip(m.captures[1]) + path[1]==path[end]=='"' && (path = path[2:end-1]) + path = joinpath(path, "plugins") + if isdir(path) + e["QT_PLUGIN_PATH"] = realpath(path) + return true + end + end + end + return false +end + +""" + pyinteract(; force=false, sleep=0.1) + +Some Python GUIs can work interactively, meaning the GUI is available but the interactive prompt is returned (e.g. after calling `matplotlib.pyplot.ion()`). +To use these from Julia, currently you must manually call `pyinteract()` each time you want to interact. + +Internally, this is calling the `PyOS_InputHook` asynchronously. Only one copy is run at a time unless `force` is true. + +The asynchronous task waits for `sleep` seconds before calling the hook function. +This gives time for the next prompt to be printed and waiting for input. +As a result, there will be a small delay before the GUI becomes interactive. +""" +pyinteract(; force::Bool=false, sleep::Real=0.1) = + if !CONFIG.inputhookrunning || force + CONFIG.inputhookrunning = true + @async begin + sleep > 0 && Base.sleep(sleep) + C.PyOS_RunInputHook() + CONFIG.inputhookrunning = false + end + nothing + end +export pyinteract + +const EVENT_LOOPS = Dict{Symbol, Base.Timer}() + +function event_loop_off(g::Symbol) + if haskey(EVENT_LOOPS, g) + Base.close(pop!(EVENT_LOOPS, g)) + end + return +end + +# function event_loop_on(g::Symbol; interval::Real=40e-3, fix::Bool=false) +# # check if already running +# if haskey(EVENT_LOOPS, g) +# return g => EVENT_LOOPS[g] +# end +# # start a new event loop +# if g in (:pyqt4, :pyqt5, :pyside, :pyside2) +# fix && fix_qt_plugin_path() +# modname = +# g == :pyqt4 ? "PyQt4" : +# g == :pyqt5 ? "PyQt5" : +# g == :pyside ? "PySide" : +# g == :pyside2 ? "PySide2" : error() +# mod = pyimport("$modname.QtCore") +# instance = mod.QCoreApplication.instance +# AllEvents = mod.QEventLoop.AllEvents +# processEvents = mod.QCoreApplication.processEvents +# maxtime = pyobject(1000*interval) +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# app = instance() +# if !pyisnone(app) +# app._in_event_loop = true +# processEvents(AllEvents, maxtime) +# end +# end +# elseif g in (:gtk, :gtk3) +# if g == :gtk3 +# gi = pyimport("gi") +# if pyisnone(gi.get_required_version("Gtk")) +# gi.require_version("Gtk", "3.0") +# end +# end +# mod = pyimport(g==:gtk ? "gtk" : g==:gtk3 ? "gi.repository.Gtk" : error()) +# events_pending = mod.events_pending +# main_iteration = mod.main_iteration +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# while pytruth(events_pending()) +# main_iteration() +# end +# end +# elseif g in (:wx,) +# mod = pyimport("wx") +# GetApp = mod.GetApp +# EventLoop = mod.EventLoop +# EventLoopActivator = mod.EventLoopActivator +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# app = GetApp() +# if !pyisnone(app) +# app._in_event_loop = true +# evtloop = EventLoop() +# ea = EventLoopActivator(evtloop) +# Pending = evtloop.Pending +# Dispatch = evtloop.Dispatch +# while pytruth(Pending()) +# Dispatch() +# end +# finalize(ea) # deactivate event loop +# app.ProcessIdle() +# end +# end +# elseif g in (:tkinter,) +# mod = pyimport("tkinter") +# _tkinter = pyimport("_tkinter") +# flag = _tkinter.ALL_EVENTS | _tkinter.DONT_WAIT +# root = PyObject(pynone) +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# new_root = mod._default_root +# if !pyisnone(new_root) +# root = new_root +# end +# if !pyisnone(root) +# while pytruth(root.dooneevent(flag)) +# end +# end +# end +# else +# error("invalid gui: $(repr(g))") +# end +# g => EVENT_LOOPS[g] +# end diff --git a/src/old/PyDict.jl b/src/old/PyDict.jl deleted file mode 100644 index 370d882d..00000000 --- a/src/old/PyDict.jl +++ /dev/null @@ -1,61 +0,0 @@ -""" - PyDict{K=PyObject, V=PyObject}(o=pydict()) - -Wrap the Python dictionary `o` (or anything satisfying the mapping interface) as a Julia dictionary with keys of type `K` and values of type `V`. -""" -struct PyDict{K,V} <: AbstractDict{K,V} - o :: PyObject - PyDict{K,V}(o::PyObject) where {K,V} = new{K,V}(o) -end -PyDict{K,V}(o=pydict()) where {K,V} = PyDict{K,V}(pydict(o)) -PyDict{K}(o=pydict()) where {K} = PyDict{K,PyObject}(o) -PyDict(o=pydict()) = PyDict{PyObject}(o) -export PyDict - -pyobject(x::PyDict) = x.o - -function Base.iterate(x::PyDict{K,V}, it=pyiter(x.o.items())) where {K,V} - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing - else - kv = pynewobject(ptr) - (pyconvert(K, kv[0]) => pyconvert(V, kv[1])), it - end -end - -Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = - (pysetitem(x.o, convert(K, k), convert(V, v)); x) - -Base.getindex(x::PyDict{K,V}, k) where {K,V} = - pyconvert(V, pygetitem(x.o, convert(K, k))) - -Base.delete!(x::PyDict{K,V}, k) where {K,V} = - (pydelitem(x.o, convert(K, k)); x) - -Base.length(x::PyDict) = Int(pylen(x.o)) - -Base.empty!(x::PyDict) = (x.o.clear(); x) - -Base.copy(x::PyDict) = typeof(x)(x.o.copy()) - -function Base.get(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : d -end - -function Base.get(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : f() -end - -function Base.get!(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = d) -end - -function Base.get!(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = f()) -end diff --git a/src/old/PyIterable.jl b/src/old/PyIterable.jl deleted file mode 100644 index f7a9585d..00000000 --- a/src/old/PyIterable.jl +++ /dev/null @@ -1,31 +0,0 @@ -""" - PyIterable{T=PyObject}(o) - -Wrap the Python object `o` into a Julia object which iterates values of type `T`. -""" -struct PyIterable{T} - o :: PyObject - PyIterable{T}(o) where {T} = new{T}(o) -end -PyIterable(o) = PyIterable{PyObject}(o) -export PyIterable - -pyobject(x::PyIterable) = x.o - -Base.length(x::PyIterable) = Int(pylen(x.o)) - -Base.IteratorSize(::Type{<:PyIterable}) = Base.SizeUnknown() - -Base.IteratorEltype(::Type{<:PyIterable}) = Base.HasEltype() - -Base.eltype(::Type{PyIterable{T}}) where {T} = T - -function Base.iterate(x::PyIterable{T}, it=pyiter(x.o)) where {T} - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing - else - (pyconvert(T, pynewobject(ptr)), it) - end -end diff --git a/src/old/base.jl b/src/old/base.jl deleted file mode 100644 index 54fc70ea..00000000 --- a/src/old/base.jl +++ /dev/null @@ -1,325 +0,0 @@ -function Base.show(io::IO, o::PyObject) - s = pyrepr(String, o) - if get(io, :typeinfo, Any) == typeof(o) - print(io, s) - elseif startswith(s, "<") && endswith(s, ">") - print(io, "") - end -end - -Base.print(io::IO, o::PyObject) = print(io, pystr(String, o)) - -function Base.show(io::IO, ::MIME"text/plain", o::PyObject) - h, w = displaysize(io) - h -= 3 - x = try - pystr_asjuliastring(pypprintmodule.pformat(o, width=w)) - catch - pyrepr(String, o) - end - if get(io, :limit, true) - if '\n' ∈ x - # multiple lines - # each one is truncated to one screen width - print(io, "py:") - h -= 1 - xs = split(x, '\n') - printlines(xs) = - for x in xs - if length(x) ≤ w - print(io, '\n', x) - else - print(io, '\n', x[1:nextind(x, 0, w-1)], '…') - end - end - if length(xs) ≤ h - # all lines fit on screen - printlines(xs) - else - # too many lines, skip the middle ones - h -= 1 - h2 = cld(h, 2) - h3 = (length(xs)+1)-(h-h2) - printlines(xs[1:h2]) - print(io, "\n ... [skipping $(h3-h2-1) lines] ...") - printlines(xs[h3:end]) - end - elseif length(x) ≤ w-4 - # one short line - print(io, "py: ", x) - return - else - # one long line - println(io, "py:") - h -= 1 - a = h * w - if length(x) ≤ a - # whole string fits on screen - print(io, x) - else - # too long, skip the middle - h -= 1 - h2 = cld(h, 2) - i2 = nextind(x, 0, h2 * w) - i3 = prevind(x, ncodeunits(x)+1, (h-h2)*w) - println(io, x[1:i2]) - println(io, " ... [skipping $(length(x[nextind(x,i2):prevind(x,i3)])) characters] ...") - print(io, x[i3:end]) - end - end - else - print(io, "py: ", x) - end -end - -Base.hasproperty(o::PyObject, k::Union{Symbol,AbstractString}) = pyhasattr(o, k) - -Base.getproperty(o::PyObject, k::Symbol) = - if k == :jl! - (T=Any) -> pyconvert(T, o) - elseif k == :jl!i - pyconvert(Int, o) - elseif k == :jl!u - pyconvert(UInt, o) - elseif k == :jl!b - pytruth(o) - elseif k == :jl!s - pystr(String, o) - elseif k == :jl!r - pyrepr(String, o) - elseif k == :jl!f - pyconvert(Cdouble, o) - elseif k == :jl!c - pyconvert(Complex{Cdouble}, o) - elseif k == :jl!list - (args...) -> PyList{args...}(o) - elseif k == :jl!dict - (args...) -> PyDict{args...}(o) - elseif k == :jl!set - (args...) -> PySet{args...}(o) - elseif k == :jl!buffer - () -> PyBuffer(o) - elseif k == :jl!array - (args...) -> PyArray{args...}(o) - elseif k == :jl!vector - (args...) -> PyVector{args...}(o) - elseif k == :jl!matrix - (args...) -> PyMatrix{args...}(o) - elseif k == :jl!pandasdf - (; opts...) -> PyPandasDataFrame(o; opts...) - elseif k == :jl!io - (; opts...) -> PyIO(o; opts...) - else - pygetattr(o, k) - end -Base.getproperty(o::PyObject, k::AbstractString) = - startswith(k, "jl!") ? getproperty(o, Symbol(k)) : pygetattr(o, k) - -Base.setproperty!(o::PyObject, k::Symbol, v) = (pysetattr(o, k, v); o) -Base.setproperty!(o::PyObject, k::AbstractString, v) = (pysetattr(o, k, v); o) - -function Base.propertynames(o::PyObject) - # this follows the logic of rlcompleter.py - function classmembers(c) - r = pydir(c) - if pyhasattr(c, "__bases__") - for b in c.__bases__ - r = pyiadd(r, classmembers(b)) - end - end - return r - end - words = pyset(pydir(o)) - words.discard("__builtins__") - if pyhasattr(o, "__class__") - words.add("__class__") - words.update(classmembers(o.__class__)) - end - r = [Symbol(pystr(String, x)) for x in words] - return r -end - - -Base.getindex(o::PyObject, k) = pygetitem(o, k) -Base.getindex(o::PyObject, k...) = pygetitem(o, k) - -Base.setindex!(o::PyObject, v, k) = (pysetitem(o, k, v); o) -Base.setindex!(o::PyObject, v, k...) = (pysetitem(o, k, v); o) - -Base.delete!(o::PyObject, k) = (pydelitem(o, k); o) - -Base.length(o::PyObject) = Int(pylen(o)) - -Base.in(v, o::PyObject) = pycontains(o, v) - -(f::PyObject)(args...; kwargs...) = pycall(f, args...; kwargs...) - -Base.eltype(::Type{T}) where {T<:PyObject} = PyObject -Base.IteratorSize(::Type{T}) where {T<:PyObject} = Base.SizeUnknown() -function Base.iterate(o::PyObject, it=pyiter(o)) - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing - else - (pynewobject(ptr), it) - end -end - -# comparisons -Base.:(==)(o1::PyObject, o2::PyObject) = pyeq(Bool, o1, o2) -Base.:(!=)(o1::PyObject, o2::PyObject) = pyne(Bool, o1, o2) -Base.:(< )(o1::PyObject, o2::PyObject) = pylt(Bool, o1, o2) -Base.:(<=)(o1::PyObject, o2::PyObject) = pyle(Bool, o1, o2) -Base.:(> )(o1::PyObject, o2::PyObject) = pygt(Bool, o1, o2) -Base.:(>=)(o1::PyObject, o2::PyObject) = pyge(Bool, o1, o2) -Base.isequal(o1::PyObject, o2::PyObject) = pyeq(Bool, o1, o2) -Base.isless(o1::PyObject, o2::PyObject) = pylt(Bool, o1, o2) - -# unary arithmetic -Base.:(-)(o::PyObject) = pyneg(o) -Base.:(+)(o::PyObject) = pypos(o) -Base.abs(o::PyObject) = pyabs(o) -Base.:(~)(o::PyObject) = pyinv(o) - -# binary arithmetic -Base.:(+)(o1::PyObject, o2::PyObject) = pyadd(o1, o2) -Base.:(-)(o1::PyObject, o2::PyObject) = pysub(o1, o2) -Base.:(*)(o1::PyObject, o2::PyObject) = pymul(o1, o2) -Base.:(/)(o1::PyObject, o2::PyObject) = pytruediv(o1, o2) -Base.:fld(o1::PyObject, o2::PyObject) = pyfloordiv(o1, o2) -Base.:mod(o1::PyObject, o2::PyObject) = pymod(o1, o2) -Base.:(^)(o1::PyObject, o2::PyObject) = pypow(o1, o2) -Base.:(<<)(o1::PyObject, o2::PyObject) = pylshift(o1, o2) -Base.:(>>)(o1::PyObject, o2::PyObject) = pyrshift(o1, o2) -Base.:(&)(o1::PyObject, o2::PyObject) = pyand(o1, o2) -Base.:xor(o1::PyObject, o2::PyObject) = pyxor(o1, o2) -Base.:(|)(o1::PyObject, o2::PyObject) = pyor(o1, o2) - -Base.:(+)(o1::PyObject, o2::Number) = pyadd(o1, o2) -Base.:(-)(o1::PyObject, o2::Number) = pysub(o1, o2) -Base.:(*)(o1::PyObject, o2::Number) = pymul(o1, o2) -Base.:(/)(o1::PyObject, o2::Number) = pytruediv(o1, o2) -Base.:fld(o1::PyObject, o2::Number) = pyfloordiv(o1, o2) -Base.:mod(o1::PyObject, o2::Number) = pymod(o1, o2) -Base.:(^)(o1::PyObject, o2::Number) = pypow(o1, o2) -Base.:(<<)(o1::PyObject, o2::Number) = pylshift(o1, o2) -Base.:(>>)(o1::PyObject, o2::Number) = pyrshift(o1, o2) -Base.:(&)(o1::PyObject, o2::Number) = pyand(o1, o2) -Base.:xor(o1::PyObject, o2::Number) = pyxor(o1, o2) -Base.:(|)(o1::PyObject, o2::Number) = pyor(o1, o2) - -Base.:(+)(o1::Number, o2::PyObject) = pyadd(o1, o2) # Defining +(::Any, ::PyObject) like this hangs Julia v1.5.2-v1.5.3 (at least) during precompilation -Base.:(-)(o1::Number, o2::PyObject) = pysub(o1, o2) -Base.:(*)(o1::Number, o2::PyObject) = pymul(o1, o2) -Base.:(/)(o1::Number, o2::PyObject) = pytruediv(o1, o2) -Base.:fld(o1::Number, o2::PyObject) = pyfloordiv(o1, o2) -Base.:mod(o1::Number, o2::PyObject) = pymod(o1, o2) -# Base.:(^)(o1::Number, o2::PyObject) = pypow(o1, o2) -# Base.:(<<)(o1::Number, o2::PyObject) = pylshift(o1, o2) -# Base.:(>>)(o1::Number, o2::PyObject) = pyrshift(o1, o2) -Base.:(&)(o1::Number, o2::PyObject) = pyand(o1, o2) -Base.:xor(o1::Number, o2::PyObject) = pyxor(o1, o2) -Base.:(|)(o1::Number, o2::PyObject) = pyor(o1, o2) - -# ternary arithmetic -Base.powermod(o1::PyObject, o2::Union{PyObject,Number}, o3::Union{PyObject,Number}) = pypow(o1, o2, o3) - -Base.zero(::Type{PyObject}) = pyint(0) -Base.one(::Type{PyObject}) = pyint(1) - -function Base.Docs.getdoc(o::PyObject) - function tryget(f, g=identity) - a = try - f() - catch - return nothing - end - pyisnone(a) ? nothing : g(a) - end - # function name(o) - # n = tryget(()->o.__name__) - # n === nothing && return nothing - # m = tryget(()->o.__module__) - # (m === nothing || string(m) == "builtins") ? string(n) : string(m, ".", n) - # end - getname(o) = tryget(()->o.__name__, string) - getdoc(o) = tryget(()->o.__doc__, string ∘ ins.cleandoc) - - docs = [] - - # Short description - ins = pyimport("inspect") - desc, name = if ins.ismodule(o).jl!b - (pyhasattr(o, "__path__") ? "package" : "module"), gname(o) - elseif ins.isgetsetdescriptor(o).jl!b - ("getset descriptor", "$(getname(o.__objclass__)).$(o.__name__)") - elseif ins.ismemberdescriptor(o).jl!b - ("member descriptor", "$(getname(o.__objclass__)).$(o.__name__)") - elseif ins.isclass(o).jl!b - ("class", getname(o)) - elseif ins.isfunction(o).jl!b || ins.isbuiltin(o).jl!b - ("function", getname(o)) - elseif ins.ismethod(o).jl!b - ("method", getname(o)) - else - ("object of type", getname(pytype(o))) - end - push!(docs, Markdown.Paragraph(["Python ", desc, " ", Markdown.Code(name), "."])) - - if ins.isroutine(o).jl!b || ins.isclass(o).jl!b - try - push!(docs, Markdown.Code("python", "$(o.__name__)$(ins.signature(o))")) - catch - end - end - - # Maybe document the class instead - doc = getdoc(o) - if doc === nothing && !ins.ismodule(o).jl!b && !ins.isclass(o).jl!b && !ins.isroutine(o).jl!b && !ins.isdatadescriptor(o).jl!b - o = pyhasattr(o, "__origin__") ? o.__origin__ : pytype(o) - doc = getdoc(o) - end - doc === nothing || push!(docs, Markdown.Paragraph([Markdown.Text(doc)])) - - # Done - Markdown.MD(docs) -end -Base.Docs.Binding(o::PyObject, k::Symbol) = getproperty(o, k) - -for (mime, method) in ((MIME"text/html", "_repr_html_"), - (MIME"text/markdown", "_repr_markdown_"), - (MIME"text/json", "_repr_json_"), - (MIME"application/javascript", "_repr_javascript_"), - (MIME"application/pdf", "_repr_pdf_"), - (MIME"image/jpeg", "_repr_jpeg_"), - (MIME"image/png", "_repr_png_"), - (MIME"image/svg+xml", "_repr_svg_"), - (MIME"text/latex", "_repr_latex_")) - T = istextmime(mime()) ? String : Vector{UInt8} - @eval begin - function Base.show(io::IO, mime::$mime, o::PyObject) - try - x = pygetattr(o, $method)() - pyisnone(x) || return write(io, pyconvert($T, x)) - catch - end - throw(MethodError(show, (io, mime, o))) - end - function Base.showable(::$mime, o::PyObject) - try - x = pygetattr(o, $method)() - if pyisnone(x) - false - else - pyconvert($T, x) - true - end - catch - false - end - end - end -end diff --git a/src/old/bool.jl b/src/old/bool.jl deleted file mode 100644 index 5d722780..00000000 --- a/src/old/bool.jl +++ /dev/null @@ -1,22 +0,0 @@ -const pybooltype = pylazyobject(() -> pybuiltins.bool) -export pybooltype - -const pytrue = pylazyobject(() -> pybuiltins.True) -const pyfalse = pylazyobject(() -> pybuiltins.False) -export pytrue, pyfalse - -pybool(args...; opts...) = pybooltype(args...; opts...) -pybool(o::Bool) = o ? pytrue : pyfalse -export pybool - -pyisbool(o::PyObject) = pytypecheck(o, pybooltype) -export pyisbool - -function pybool_tryconvert(::Type{T}, o::PyObject) where {T} - x = pytruth(o) - if Bool <: T - pytruth(o) - else - tryconvert(T, x) - end -end diff --git a/src/old/builtins.jl b/src/old/builtins.jl deleted file mode 100644 index dea7aba1..00000000 --- a/src/old/builtins.jl +++ /dev/null @@ -1,60 +0,0 @@ -const pybuiltins = pylazyobject(() -> pyimport("builtins")) -export pybuiltins - -# types -for p in [:classmethod, :enumerate, :filter, :map, :property, :reversed, :staticmethod, :super, :zip] - j = Symbol(:py, p, :type) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - @eval export $j -end - -# functions -for p in [:all, :any, :chr, :compile, :eval, :exec, :format, :help, :hex, :id, :max, :min, :next, :oct, :open, :ord, :print, :round, :sorted, :sum, :vars] - j = Symbol(:py, p, :func) - jf = Symbol(:py, p) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - if p in [:help, :print, :exec] - @eval $jf(args...; opts...) = ($j(args...; opts...); nothing) - else - @eval $jf(args...; opts...) = $j(args...; opts...) - end - @eval export $jf -end - -# singletons -for p in [:Ellipsis, :NotImplemented] - j = Symbol(:py, lowercase(string(p))) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - @eval export $j -end - -# exceptions and warnings -# NOTE: We import these directly from C, so that the error indicator is not inadvertantly -# reset when the lazy object is evaluated the first time (otherwise there can be problems -# using `pyerroccurred(t)`). -for n in [ - # exceptions - :BaseException, :Exception, :ArithmeticError, :AssertionError, :AttributeError, - :BlockingIOError, :BrokenPipeError, :BufferError, :ChildProcessError, - :ConnectionAbortedError, :ConnectionError, :ConnectionRefusedError, - :ConnectionResetError, :EOFError, :FileExistsError, :FileNotFoundError, - :FloatingPointError, :GeneratorExit, :ImportError, :IndentationError, :IndexError, - :InterruptedError, :IsADirectoryError, :KeyError, :KeyboardInterrupt, :LookupError, - :MemoryError, :ModuleNotFoundError, :NameError, :NotADirectoryError, - :NotImplementedError, :OSError, :OverflowError, :PermissionError, :ProcessLookupError, - :RecursionError, :ReferenceError, :RuntimeError, :StopAsyncIteration, :StopIteration, - :SyntaxError, :SystemError, :SystemExit, :TabError, :TimeoutError, :TypeError, - :UnboundLocalError, :UnicodeDecodeError, :UnicodeEncodeError, :UnicodeError, - :UnicodeTranslateError, :ValueError, :ZeroDivisionError, - # aliases - :EnvironmentError, :IOError, :WindowsError, - # warnings - :Warning, :BytesWarning, :DeprecationWarning, :FutureWarning, :ImportWarning, - :PendingDeprecationWarning, :ResourceWarning, :RuntimeWarning, :SyntaxWarning, - :UnicodeWarning, :UserWarning, -] - j = Symbol(:py, lowercase(string(n))) - p = QuoteNode(Symbol(:PyExc_, n)) - @eval const $j = pylazyobject(() -> pyborrowedobject(unsafe_load(Ptr{CPyPtr}(C.pyglobal($p))))) - @eval export $j -end diff --git a/src/old/bytearray.jl b/src/old/bytearray.jl deleted file mode 100644 index 705a6fa8..00000000 --- a/src/old/bytearray.jl +++ /dev/null @@ -1,8 +0,0 @@ -const pybytearraytype = pylazyobject(() -> pybuiltins.bytearray) -export pybytearraytype - -pyisbytearray(o::PyObject) = pytypecheck(o, pybytearraytype) -export pyisbytearray - -pybytearray(args...; opts...) = pybytearraytype(args...; opts...) -export pybytearray diff --git a/src/old/bytes.jl b/src/old/bytes.jl deleted file mode 100644 index 923005c5..00000000 --- a/src/old/bytes.jl +++ /dev/null @@ -1,16 +0,0 @@ -const pybytestype = pylazyobject(() -> pybuiltins.bytes) -export pybytestype - -pyisbytes(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_BYTES_SUBCLASS) -export pyisbytes - -pybytes(args...; opts...) = pybytestype(args...; opts...) -pybytes(x::Vector{UInt8}) = check(C.PyBytes_FromStringAndSize(pointer(x), length(x))) -export pybytes - -pybytes_asjuliastring(o::PyObject) = GC.@preserve o begin - buf = Ref{Ptr{Cchar}}() - len = Ref{C.Py_ssize_t}() - check(C.PyBytes_AsStringAndSize(o, buf, len)) - unsafe_string(buf[], len[]) -end diff --git a/src/old/collections.jl b/src/old/collections.jl deleted file mode 100644 index 375afb7b..00000000 --- a/src/old/collections.jl +++ /dev/null @@ -1,146 +0,0 @@ -for p in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, :Sized, :Callable, :Collection, :Sequence, :MutableSequence, :ByteString, :Set, :MutableSet, :Mapping, :MutableMapping, :MappingView, :ItemsView, :KeysView, :ValuesView, :Awaitable, :Coroutine, :AsyncIterable, :AsyncIterator, :AsyncGenerator] - j = Symbol(:py, lowercase(string(p)), :abc) - @eval const $j = pylazyobject(() -> pycollectionsabcmodule.$p) - @eval export $j -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T} - # check subclasses - if pyisinstance(o, pymappingabc) - r = pymapping_tryconvert(T, o) - r === PyConvertFail() || return r - elseif pyisinstance(o, pysequenceabc) - r = pysequence_tryconvert(T, o) - r === PyConvertFail() || return r - elseif pyisinstance(o, pysetabc) - r = pyabstractset_tryconvert(T, o) - r === PyConvertFail() || return r - end - # generic conversions - if (S = _typeintersect(T, PyIterable)) != Union{} - return S(o) - elseif (S = _typeintersect(T, Vector{PyObject})) != Union{} - r = S() - append!(r, PyIterable{PyObject}(o)) - elseif (S = _typeintersect(T, Vector)) != Union{} - r = S() - append!(r, PyIterable{eltype(r)}(o)) - elseif (S = _typeintersect(T, Set{PyObject})) != Union{} - r = S() - append!(r, PyIterable{PyObject}(o)) - elseif (S = _typeintersect(T, Set)) != Union{} - r = S() - append!(r, PyIterable{eltype(r)}(o)) - elseif (S = _typeintersect(T, Tuple)) != Union{} - pyiterable_tryconvert(S, o) - elseif (S = _typeintersect(T, CartesianIndex)) != Union{} - pyiterable_tryconvert(S, o) - elseif (S = _typeintersect(T, NamedTuple)) != Union{} - pyiterable_tryconvert(S, o) - else - tryconvert(T, PyIterable(o)) - end -end - -# TODO: make parts of this generated for type stability (i.e. the parts dealing with type logic) -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:Tuple} - # union? - if T isa Union - a = pyiterable_tryconvert(T.a, o) - b = pyiterable_tryconvert(T.b, o) - if typeof(a) == typeof(b) - return a - elseif a === PyConvertFail() - return b - elseif b === PyConvertFail() - return a - else - error("ambiguous conversion") - end - end - # flatten out any type vars - S = _type_flatten_tuple(T) :: DataType - # determine component types - if length(S.parameters) > 0 && Base.isvarargtype(S.parameters[end]) - nfixed = length(S.parameters) - 1 - vartype = S.parameters[end].body.parameters[1] - else - nfixed = length(S.parameters) - vartype = nothing - end - # convert components - xs = [] - for (i,xo) in enumerate(o) - if i ≤ nfixed - x = pytryconvert(S.parameters[i], xo) - x === PyConvertFail() ? (return x) : push!(xs, x) - elseif vartype === nothing - # too many values - return PyConvertFail() - else - x = pytryconvert(vartype, xo) - x === PyConvertFail() ? (return x) : push!(xs, x) - end - end - # check we got enough - length(xs) ≥ nfixed || return PyConvertFail() - # success! - tryconvert(T, Tuple(xs)) -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:CartesianIndex} - x = pyiterable_tryconvert(Tuple{Vararg{Int}}, o) - x === PyConvertFail() ? x : convert(T, CartesianIndex(x)) -end - -function pyiterable_tryconvert(::Type{CartesianIndex{N}}, o::PyObject) where {N} - x = pyiterable_tryconvert(NTuple{N,Int}, o) - x === PyConvertFail() ? x : CartesianIndex{N}(x) -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:NamedTuple} - PyConvertFail() -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {names, T<:NamedTuple{names}} - x = pyiterable_tryconvert(NTuple{length(names), PyObject}, o) - x === PyConvertFail() ? x : convert(T, NamedTuple{names}(x)) -end - -function pyiterable_tryconvert(::Type{NamedTuple{names,Ts}}, o::PyObject) where {names, Ts<:Tuple} - x = pyiterable_tryconvert(Ts, o) - x === PyConvertFail() ? x : NamedTuple{names,Ts}(x) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, AbstractDict)) != Union{} - pymapping_tryconvert(S, o) - else - PyConvertFail() - end -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {T<:AbstractDict} - tryconvert(T, PyDict(o)) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {K, T<:AbstractDict{K}} - # often fails because e.g. convert(::Dict{String}, ::PyDict{String,Int}) fails - tryconvert(T, PyDict{K}(o)) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {K, V, T<:AbstractDict{K,V}} - tryconvert(T, PyDict{K,V}(o)) -end - -function pysequence_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, PyList)) != Union{} - S(o) - else - PyConvertFail() - end -end - -function pyabstractset_tryconvert(::Type{T}, o::PyObject) where {T} - PyConvertFail() -end diff --git a/src/old/complex.jl b/src/old/complex.jl deleted file mode 100644 index dbfcf5be..00000000 --- a/src/old/complex.jl +++ /dev/null @@ -1,24 +0,0 @@ -const pycomplextype = pylazyobject(() -> pybuiltins.complex) -export pycomplextype - -pycomplex(args...; opts...) = pycomplextype(args...; opts...) -pycomplex(x::Complex) = pycomplex(real(x), imag(x)) -export pycomplex - -pyiscomplex(o::PyObject) = pytypecheck(o, pycomplextype) -export pyiscomplex - -function pycomplex_tryconvert(::Type{T}, o::PyObject) where {T} - x = check(C.PyComplex_RealAsDouble(o), true) - y = check(C.PyComplex_ImagAsDouble(o), true) - z = Complex(x, y) - if (S = _typeintersect(T, Complex{Cdouble})) != Union{} - convert(S, z) - elseif (S = _typeintersect(T, Complex)) != Union{} - tryconvert(S, z) - elseif (S = _typeintersect(T, Real)) != Union{} - tryconvert(S, z) - else - tryconvert(T, z) - end -end diff --git a/src/old/convert.jl b/src/old/convert.jl deleted file mode 100644 index 57575635..00000000 --- a/src/old/convert.jl +++ /dev/null @@ -1,221 +0,0 @@ -function pyconvert(::Type{T}, o::PyObject) where {T} - r = pytryconvert(T, o) - r === PyConvertFail() ? error("cannot convert this Python `$(pytype(o).__name__)` to a Julia `$T`") : r -end -pyconvert(::Type{T}) where {T} = o -> pyconvert(T, o) -export pyconvert - -function pytryconvert(::Type{T}, o::PyObject) where {T} - # special cases - if T == PyObject - return PyObject(o) - elseif pyisjl(o) - return tryconvert(T, pyjlgetvalue(o)) - end - - # types - for t in pytype(o).__mro__ - n = "$(t.__module__).$(t.__name__)" - c = get(PYTRYCONVERT_TYPE_RULES, n, (T,o)->PyConvertFail()) - r = c(T, o) :: Union{T, PyConvertFail} - r === PyConvertFail() || return r - end - - # buffer/array interface - if (C.PyObject_CheckBuffer(o) != 0) || (pyhasattr(o, "__array__") && !pyisnone(o.__array__)) || (pyhasattr(o, "__array_interface__") && !pyisnone(o.__array_interface__)) - r = tryconvert(T, PyArray(o)) - r === PyConvertFail() || return r - end - # iterable interface (includes sequences, mappings and sets) - if pyisinstance(o, pyiterableabc) - r = pyiterable_tryconvert(T, o) :: Union{T, PyConvertFail} - r === PyConvertFail() || return r - end - # TODO: IO interface - # TODO: number interface - - # so that T=Any always succeeds - if o isa T - return o - end - - # failure - return PyConvertFail() -end -pytryconvert(::Type{T}) where {T} = o -> pytryconvert(T, o) -export pytryconvert - -const PYTRYCONVERT_TYPE_RULES = Dict{String,Function}( - "builtins.NoneType" => pynone_tryconvert, - "builtins.bool" => pybool_tryconvert, - "builtins.str" => pystr_tryconvert, - "builtins.int" => pyint_tryconvert, - "builtins.float" => pyfloat_tryconvert, - "builtins.complex" => pycomplex_tryconvert, - "builtins.Fraction" => pyfraction_tryconvert, - "builtins.range" => pyrange_tryconvert, - "builtins.tuple" => pytuple_tryconvert, - "pandas.core.frame.DataFrame" => pypandasdataframe_tryconvert, - # TODO: datetime, date, time - # NOTE: we don't need to include standard containers here because we can access them via standard interfaces (Sequence, Mapping, Buffer, etc.) -) - -Base.convert(::Type{T}, o::PyObject) where {T} = pyconvert(T, o) -Base.convert(::Type{Any}, o::PyObject) = o - -### SPECIAL CONVERSIONS - -@generated _eltype(o) = try eltype(o); catch; missing; end -_eltype(o::Type) = missing - -@generated _keytype(o) = try keytype(o); catch; missing; end -_keytype(o::Base.RefValue) = Tuple{} -_keytype(o::NamedTuple) = Union{Symbol,Int} -_keytype(o::Tuple) = Int -_keytype(o::Type) = missing - -@generated _valtype(o, k...) = try valtype(o); catch; missing; end -_valtype(o::NamedTuple, k::Int) = fieldtype(typeof(o), k) -_valtype(o::NamedTuple, k::Symbol) = fieldtype(typeof(o), k) -_valtype(o::Tuple, k::Int) = fieldtype(typeof(o), k) -_valtype(o::Type) = missing - -hasmultiindex(o) = true -hasmultiindex(o::Type) = true -hasmultiindex(o::AbstractDict) = false -hasmultiindex(o::NamedTuple) = false -hasmultiindex(o::Tuple) = false - -""" - pyconvert_element(o, v::PyObject) - -Convert `v` to be of the right type to be an element of `o`. -""" -pytryconvert_element(o, v) = pytryconvert(_eltype(o)===missing ? Any : _eltype(o), v) -pyconvert_element(args...) = - let r = pytryconvert_element(args...) - r === PyConvertFail() ? error("cannot convert this to an element") : r - end - -""" - pytryconvert_indices(o, k::PyObject) - -Convert `k` to a tuple of indices for `o`. -""" -function pytryconvert_indices(o, k) - if _keytype(o) !== missing - i = pytryconvert(_keytype(o), k) - i === PyConvertFail() ? PyConvertFail() : (i,) - elseif hasmultiindex(o) && pyistuple(k) - Tuple(pyconvert(Any, x) for x in k) - else - (pyconvert(Any, k),) - end -end -pyconvert_indices(args...) = - let r = pytryconvert_indices(args...) - r === PyConvertFail() ? error("cannot convert this to indices") : r - end - -""" - pyconvert_value(o, v::PyObject, k...) - -Convert `v` to be of the right type to be a value of `o` at indices `k`. -""" -pytryconvert_value(o, v, k...) = pytryconvert(_valtype(o)===missing ? Any : _valtype(o), v) -pyconvert_value(args...) = - let r = pytryconvert_value(args...) - r === PyConvertFail() ? error("cannot convert this to a value") : r - end - -""" - pyconvert_args(NamedTuple{argnames, argtypes}, args, kwargs=nothing; defaults=nothing, numpos=len(names), numposonly=0) - -Parse the Python tuple `args` and optionally Python dict `kwargs` as function arguments, returning a `NamedTuple{names,types}`. - -- `defaults` is a named tuple of default parameter values for optional arguments. -- `numpos` is the number of possibly-positional arguments. -- `numposonly` is the number of positional-only arguments. -""" -@generated function pyconvert_args( - ::Type{NamedTuple{argnames,argtypes}}, - args::PyObject, - kwargs::Union{PyObject,Nothing}=nothing; - defaults::Union{NamedTuple,Nothing}=nothing, - numposonly::Integer=0, - numpos::Integer=length(argnames), -) where {argnames, argtypes} - code = [] - argvars = [] - push!(code, quote - 0 ≤ numposonly ≤ numpos ≤ length(argnames) || error("require 0 ≤ numposonly ≤ numpos ≤ numargs") - nargs = pylen(args) - nargs ≤ numpos || pythrow(pytypeerror("at most $numpos positional arguments expected, got $nargs")) - end) - for (i,(argname,argtype)) in enumerate(zip(argnames,argtypes.parameters)) - argvar = gensym() - push!(argvars, argvar) - push!(code, quote - $argvar = - if $i ≤ numpos && $i ≤ nargs - kwargs !== nothing && $(string(argname)) in kwargs && pythrow(pytypeerror($("argument '$argname' got multiple values"))) - let r = pytryconvert($argtype, args[$(i-1)]) - r===PyConvertFail() ? pythrow(pytypeerror($("argument '$argname' has wrong type"))) : r - end - elseif kwargs !== nothing && $i > numposonly && $(string(argname)) in kwargs - let r = pytryconvert($argtype, kwargs.pop($(string(argname)))) - r===PyConvertFail() ? pythrow(pytypeerror($("argument '$argname' has wrong type"))) : r - end - elseif defaults !== nothing && haskey(defaults, $(QuoteNode(argname))) - convert($argtype, defaults[$(QuoteNode(argname))]) - else - pythrow(pytypeerror($("argument '$argname' not given"))) - end - end) - end - push!(code, quote - kwargs !== nothing && pylen(kwargs) > 0 && pythrow(pytypeerror("invalid keyword arguments: $(join(kwargs, ", "))")) - NamedTuple{$argnames, $argtypes}(($(argvars...),)) - end) - ex = Expr(:block, code...) -end - -macro pyconvert_args(spec, args, kwargs=nothing) - spec isa Expr && spec.head == :tuple || @goto error - argnames = [] - argtypes = [] - defaults = [] - numposonly = 0 - numpos = -1 - for argspec in spec.args - if argspec === :* - numpos = length(argnames) - continue - elseif argspec == :/ - numposonly = length(argnames) - continue - end - if argspec isa Expr && argspec.head == :(=) - argspec, dflt = argspec.args - else - dflt = nothing - end - if argspec isa Expr && argspec.head == :(::) - argspec, tp = argspec.args - else - tp = PyObject - end - if argspec isa Symbol - nm = argspec - else - @goto error - end - push!(argnames, nm) - push!(argtypes, tp) - dflt === nothing || push!(defaults, nm => dflt) - end - numpos < 0 && (numpos = length(argnames)) - return :(pyconvert_args(NamedTuple{($(map(QuoteNode, argnames)...),),Tuple{$(map(esc, argtypes)...)}}, $(esc(args)), $(esc(kwargs)), defaults=NamedTuple{($([QuoteNode(n) for (n,d) in defaults]...),)}(($([esc(d) for (n,d) in defaults]...),)), numpos=$numpos, numposonly=$numposonly)) - @label error - error("invalid argument specification") -end diff --git a/src/old/dict.jl b/src/old/dict.jl deleted file mode 100644 index 05bdaff9..00000000 --- a/src/old/dict.jl +++ /dev/null @@ -1,34 +0,0 @@ -const pydicttype = pylazyobject(() -> pybuiltins.dict) -export pydicttype - -pyisdict(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_DICT_SUBCLASS) -export pyisdict - -pydict(args...; opts...) = pydicttype(args...; opts...) -pydict() = check(C.PyDict_New()) -pydict(o::Union{AbstractDict, Base.Iterators.Pairs, AbstractArray{<:Pair}}) = pydict_fromiter(o) -pydict(o::NamedTuple) = pydict_fromiter(pairs(o)) -export pydict - -function pydict_fromiter(kvs) - d = pydict() - for (k,v) in kvs - vo = pyobject(v) - if k isa AbstractString - check(C.PyDict_SetItemString(d, k, vo)) - else - check(C.PyDict_SetItem(d, pyobject(k), vo)) - end - end - return d -end - -function pydict_fromstringiter(kvs) - d = pydict() - for (k,v) in kvs - ko = k isa AbstractString ? k : k isa Symbol ? String(k) : error("only string and symbols allowed") - vo = pyobject(v) - check(C.PyDict_SetItemString(d, ko, vo)) - end - return d -end diff --git a/src/old/error.jl b/src/old/error.jl deleted file mode 100644 index 640a993f..00000000 --- a/src/old/error.jl +++ /dev/null @@ -1,136 +0,0 @@ -check(::Type{CPyPtr}, o::CPyPtr, ambig=false) = (o == C_NULL && (ambig ? pyerrcheck() : pythrow()); o) -check(::Type{T}, o::T, ambig=false) where {T<:Number} = (o == (zero(T)-one(T)) && (ambig ? pyerrcheck() : pythrow()); o) -check(::Type{Nothing}, o::Cvoid, ambig=false) = ambig ? pyerrcheck() : nothing -check(::Type{PyObject}, o::CPyPtr, ambig=false) = pynewobject(check(CPyPtr, o, ambig)) -check(::Type{Bool}, o::Cint, ambig=false) = !iszero(check(Cint, o, ambig)) -check(::Type{Nothing}, o::Cint, ambig=false) = (check(Cint, o, ambig); nothing) -check(o::CPyPtr, ambig=false) = check(PyObject, o, ambig) -check(o::T, ambig=false) where {T<:Number} = check(T, o, ambig) -check(o::Cvoid, ambig=false) = check(Nothing, o, ambig) - - -pyerroccurred() = C.PyErr_Occurred() != C_NULL - -function pyerroccurred(t::PyObject) - e = C.PyErr_Occurred() - e != C_NULL && !iszero(C.PyErr_GivenExceptionMatches(e, t)) -end - -pyerrclear() = C.PyErr_Clear() - -pyerrcheck() = pyerroccurred() ? pythrow() : nothing - -pyerrset(t::PyObject) = C.PyErr_SetNone(t) -pyerrset(t::PyObject, x::AbstractString) = C.PyErr_SetString(t, x) -pyerrset(t::PyObject, x::PyObject) = C.PyErr_SetObject(t, x) - -pyerrmatches(e::PyObject, t::PyObject) = !iszero(C.PyErr_GivenExceptionMatches(e, t)) - -struct PythonRuntimeError <: Exception - t :: PyObject - v :: PyObject - b :: PyObject -end - -function pyerrfetch(normalize::Bool=false) - t = Ref{CPyPtr}() - v = Ref{CPyPtr}() - b = Ref{CPyPtr}() - C.PyErr_Fetch(t, v, b) - normalize && C.PyErr_NormalizeException(t, v, b) - to = t[]==C_NULL ? pynone : pynewobject(t[]) - vo = v[]==C_NULL ? pynone : pynewobject(v[]) - bo = b[]==C_NULL ? pynone : pynewobject(b[]) - (to, vo, bo) -end - -pyerrrestore(t::PyObject, v::PyObject, b::PyObject) = - C.PyErr_Restore(pyincref!(t), pyincref!(v), pyincref!(b)) - -pyerrrestore(err::PythonRuntimeError) = pyerrrestore(err.t, err.v, err.b) - -pythrow() = throw(PythonRuntimeError(pyerrfetch(true)...)) - -pythrow(v::PyObject) = - throw(PythonRuntimeError(pytype(v), v, pynone)) - -function Base.showerror(io::IO, e::PythonRuntimeError) - if pyisnone(e.t) - print(io, "Python: mysterious error (no error was actually set)") - return - end - - if CONFIG.sysautolasttraceback - try - pysysmodule.last_type = e.t - pysysmodule.last_value = e.v - pysysmodule.last_traceback = e.b - catch err - print(io, "") - end - end - - # if this is a Julia exception then recursively print it and its stacktrace - if pyerrmatches(e.t, pyjlexception) - try - jp = pyjlgetvalue(e.v.args[0]) - if jp !== nothing - je, jb = jp - print(io, "Python: Julia: ") - showerror(io, je) - if jb === nothing - println(io) - print(io, "Stacktrace: none") - else - io2 = IOBuffer() - Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) - printstyled(io, String(take!(io2))) - end - end - if pyisnone(e.b) - println(io) - printstyled(io, "Python stacktrace: none") - return - else - @goto pystacktrace - end - catch err - println(io, "") - end - end - - # otherwise, print the Python exception - print(io, "Python: ") - try - print(io, e.t.__name__) - catch err - print(io, "") - end - if !pyisnone(e.v) - print(io, ": ") - try - print(io, e.v) - catch err - print(io, "") - end - end - if !pyisnone(e.b) - @label pystacktrace - println(io) - printstyled(io, "Python stacktrace:") - try - fs = pytracebackmodule.extract_tb(e.b) - nfs = pylen(fs) - for i in 1:nfs - println(io) - f = fs[nfs-i] - printstyled(io, " [", i, "] ", ) - printstyled(io, f.name, bold=true) - printstyled(io, " at ") - printstyled(io, f.filename, ":", f.lineno, bold=true) - end - catch err - print(io, "") - end - end -end diff --git a/src/old/eval.jl b/src/old/eval.jl deleted file mode 100644 index 0e28a9bc..00000000 --- a/src/old/eval.jl +++ /dev/null @@ -1,137 +0,0 @@ -const SCOPES = Dict{Any,PyObject}() -const COMPILECACHE = Dict{Tuple{String,String,String},PyObject}() - -scope(s) = get!(pydict, SCOPES, s) -scope(s::PyObject) = s - -""" - pyeval(src, scope, locals=nothing) - -Evaluate Python expression `src` in the context of the given `scope` and `locals`. Return the value. - -If the `scope` is a Python object, it must be a `dict` to use as globals. - -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyeval(src, @__MODULE__)` evaluates in a scope unique to the current module. -""" -pyeval(src, globals, locals=nothing) = pyevalfunc(src, scope(globals), locals) - -""" - pyexec(src, scope, locals=nothing) - -Execute Python expression `src` in the context of the given `scope` and `locals`. - -If the `scope` is a Python object, it must be a `dict` to use as globals. - -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyexec(src, @__MODULE__)` executes in a scope unique to the current module. -""" -pyexec(src, globals, locals=nothing) = (pyexecfunc(src, scope(globals), locals); nothing) - -""" - pyevaldb(src, scope, locals=nothing) - -Same as `pyeval(src, scope, locals)` but evaluated inside a `pdb` debugger. -""" -pyevaldb(src, globals, locals=nothing) = pypdbmodule.runeval(src, scope(globals), locals) - -""" - pyexecdb(src, scope, locals=nothing) - -Same as `pyexec(src, scope, locals)` but evaluated inside a `pdb` debugger. -""" -pyexecdb(src, globals, locals=nothing) = (pypdbmodule.run(src, scope(globals), locals); nothing) - -""" - py"...."[flags] - -Evaluate (`v`) or execute (`x`) the given Python source code. - -Julia values may be interpolated into the source code with `\$` syntax. For a literal `\$`, enter `\$\$`. - -Execution occurs in a global scope unique to the current module. - -The flags can be any combination of the following characters: -- `v` (evaluate): Evaluate a single expression and return its value. -- `x` (execute): Execute the code and return `nothing`. -- `g` (globals): Return the dict of globals for the scope instead. -- `l` (locals): Perform the computation in a new local scope and return that scope. -- `c` (compile): Cache a compiled version of the source and re-use it each time, for speed. -- `d` (debug): Run inside a `pdb` debugger. - -If neither `v` nor `x` is specified and the code is a single line then `v` (evaluate) is assumed, otherwise `x` (execute). -""" -macro py_str(src::String, flags::String="") - # parse the flags - exec = '\n' in src - retglobals = false - retlocals = false - lazy = false - compile = false - debug = false - for f in flags - if f == 'x' - exec = true - elseif f == 'v' - exec = false - elseif f == 'g' - retglobals = true - elseif f == 'l' - retlocals = true - # elseif f == 'z' - # lazy = true - elseif f == 'c' - compile = true - elseif f == 'd' - debug = true - else - error("invalid flags: `py\"...\"$flags`") - end - end - # parse src for $-interpolations - chunks = String[] - interps = [] - i = firstindex(src) - while true - j = findnext('$', src, i) - if j === nothing - push!(chunks, src[i:end]) - break - else - push!(chunks, src[i:prevind(src,j)]) - end - if checkbounds(Bool, src, j+1) && src[j+1] == '$' - push!(chunks, "\$") - i = j+2 - else - ex, i = Meta.parse(src, j+1, greedy=false) - var = "_jl_interp_$(length(interps)+1)_" - push!(interps, var => ex) - push!(chunks, "($var)") - end - end - newsrcstr = newsrc = join(chunks) - # compile the code so there is string information - cfile = "julia:$(__source__.file):$(__source__.line)" - cmode = exec ? "exec" : "eval" - newsrc = :(pycompile($newsrcstr, $cfile, $cmode)) - if compile - # compile the code lazily (so the python parser is only invoked once) - # Julia crashes if you try to put pylazyobject(()->pycompile(...)) directly in the syntax tree. Is this a bug?? - # Instead, we use a run-time lookup table. - newsrc = :(get!(()->$newsrc, COMPILECACHE, ($newsrcstr, $cfile, $cmode))) - end - # make the expression - ex = :(let - globals = scope(@__MODULE__) - locals = $(retlocals ? :(pydict()) : :globals) - $([:(check(C.PyDict_SetItemString(globals, $k, pyobject($(esc(v)))))) for (k,v) in interps]...) - result = $(debug ? (exec ? :pyexecdb : :pyevaldb) : (exec ? :pyexec : :pyeval))($newsrc, globals, locals) - $([:(check(C.PyDict_DelItemString(globals, $k))) for (k,v) in interps]...) - $(retlocals ? :locals : retglobals ? :globals : exec ? nothing : :result) - end) - if lazy - # wrap as a lazy object - ex = :(pylazyobject(() -> $ex)) - end - ex -end -export @py_str diff --git a/src/old/float.jl b/src/old/float.jl deleted file mode 100644 index 0dce1b6f..00000000 --- a/src/old/float.jl +++ /dev/null @@ -1,22 +0,0 @@ -const pyfloattype = pylazyobject(() -> pybuiltins.float) -export pyfloattype - -pyfloat(args...; opts...) = pyfloattype(args...; opts...) -pyfloat(x::Real) = check(C.PyFloat_FromDouble(x)) -export pyfloat - -pyisfloat(o::PyObject) = pytypecheck(o, pyfloattype) -export pyisfloat - -function pyfloat_tryconvert(::Type{T}, o::PyObject) where {T} - x = check(C.PyFloat_AsDouble(o), true) - if (S = _typeintersect(T, Cdouble)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, AbstractFloat)) != Union{} - tryconvert(S, x) - elseif (S = _typeintersect(T, Real)) != Union{} - tryconvert(S, x) - else - tryconvert(T, x) - end -end diff --git a/src/old/fraction.jl b/src/old/fraction.jl deleted file mode 100644 index c28e31e6..00000000 --- a/src/old/fraction.jl +++ /dev/null @@ -1,33 +0,0 @@ -const pyfractiontype = pylazyobject(() -> pyfractionsmodule.Fraction) -export pyfractiontype - -pyfraction(args...; opts...) = pyfractiontype(args...; opts...) -pyfraction(x::Rational) = pyfraction(numerator(x), denominator(x)) -export pyfraction - -pyisfraction(o::PyObject) = pytypecheck(o, pyfractiontype) -export pyisfraction - -function pyfraction_tryconvert(::Type{T}, o::PyObject) where {T<:Rational} - tryconvert(T, pyfraction_tryconvert(Rational{BigInt}, o)) -end - -function pyfraction_tryconvert(::Type{Rational{T}}, o::PyObject) where {T<:Integer} - x = pyint_tryconvert(T, o.numerator) - x === PyConvertFail() && return x - y = pyint_tryconvert(T, o.denominator) - y === PyConvertFail() && return y - Rational{T}(x, y) -end - -function pyfraction_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, Rational)) != Union{} - pyfraction_tryconvert(S, o) - elseif (S = _typeintersect(T, Integer)) != Union{} - o.denominator == pyint(1) ? pyint_tryconvert(S, o.numerator) : PyConvertFail() - elseif (S = _typeintersect(T, Number)) != Union{} - tryconvert(S, pyfraction_tryconvert(Rational{BigInt}, o)) - else - tryconvert(T, pyfraction_tryconvert(Rational{BigInt}, o)) - end -end diff --git a/src/old/gui.jl b/src/old/gui.jl deleted file mode 100644 index c98d1d78..00000000 --- a/src/old/gui.jl +++ /dev/null @@ -1,145 +0,0 @@ -""" - fix_qt_plugin_path() - -Try to set the `QT_PLUGIN_PATH` environment variable in Python, if not already set. - -This fixes the problem that Qt does not know where to find its `qt.conf` file, because it -always looks relative to `sys.executable`, which can be the Julia executable not the Python -one when using this package. - -If `CONFIG.qtfix` is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide` or `PySide2` are imported. -""" -function fix_qt_plugin_path() - e = pyosmodule.environ - "QT_PLUGIN_PATH" in e && return false - CONFIG.exepath === nothing && return false - qtconf = joinpath(dirname(CONFIG.exepath), "qt.conf") - isfile(qtconf) || return false - for line in eachline(qtconf) - m = match(r"^\s*prefix\s*=(.*)$"i, line) - if m !== nothing - path = strip(m.captures[1]) - path[1]==path[end]=='"' && (path = path[2:end-1]) - path = joinpath(path, "plugins") - if isdir(path) - e["QT_PLUGIN_PATH"] = realpath(path) - return true - end - end - end - return false -end - -""" - pyinteract(; force=false, sleep=0.1) - -Some Python GUIs can work interactively, meaning the GUI is available but the interactive prompt is returned (e.g. after calling `matplotlib.pyplot.ion()`). -To use these from Julia, currently you must manually call `pyinteract()` each time you want to interact. - -Internally, this is calling the `PyOS_InputHook` asynchronously. Only one copy is run at a time unless `force` is true. - -The asynchronous task waits for `sleep` seconds before calling the hook function. -This gives time for the next prompt to be printed and waiting for input. -As a result, there will be a small delay before the GUI becomes interactive. -""" -pyinteract(; force::Bool=false, sleep::Real=0.1) = - if !CONFIG.inputhookrunning || force - CONFIG.inputhookrunning = true - @async begin - sleep > 0 && Base.sleep(sleep) - C.PyOS_RunInputHook() - CONFIG.inputhookrunning = false - end - nothing - end -export pyinteract - -const EVENT_LOOPS = Dict{Symbol, Base.Timer}() - -function event_loop_off(g::Symbol) - if haskey(EVENT_LOOPS, g) - Base.close(pop!(EVENT_LOOPS, g)) - end - return -end - -function event_loop_on(g::Symbol; interval::Real=40e-3, fix::Bool=false) - # check if already running - if haskey(EVENT_LOOPS, g) - return g => EVENT_LOOPS[g] - end - # start a new event loop - if g in (:pyqt4, :pyqt5, :pyside, :pyside2) - fix && fix_qt_plugin_path() - modname = - g == :pyqt4 ? "PyQt4" : - g == :pyqt5 ? "PyQt5" : - g == :pyside ? "PySide" : - g == :pyside2 ? "PySide2" : error() - mod = pyimport("$modname.QtCore") - instance = mod.QCoreApplication.instance - AllEvents = mod.QEventLoop.AllEvents - processEvents = mod.QCoreApplication.processEvents - maxtime = pyobject(1000*interval) - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - app = instance() - if !pyisnone(app) - app._in_event_loop = true - processEvents(AllEvents, maxtime) - end - end - elseif g in (:gtk, :gtk3) - if g == :gtk3 - gi = pyimport("gi") - if pyisnone(gi.get_required_version("Gtk")) - gi.require_version("Gtk", "3.0") - end - end - mod = pyimport(g==:gtk ? "gtk" : g==:gtk3 ? "gi.repository.Gtk" : error()) - events_pending = mod.events_pending - main_iteration = mod.main_iteration - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - while pytruth(events_pending()) - main_iteration() - end - end - elseif g in (:wx,) - mod = pyimport("wx") - GetApp = mod.GetApp - EventLoop = mod.EventLoop - EventLoopActivator = mod.EventLoopActivator - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - app = GetApp() - if !pyisnone(app) - app._in_event_loop = true - evtloop = EventLoop() - ea = EventLoopActivator(evtloop) - Pending = evtloop.Pending - Dispatch = evtloop.Dispatch - while pytruth(Pending()) - Dispatch() - end - finalize(ea) # deactivate event loop - app.ProcessIdle() - end - end - elseif g in (:tkinter,) - mod = pyimport("tkinter") - _tkinter = pyimport("_tkinter") - flag = _tkinter.ALL_EVENTS | _tkinter.DONT_WAIT - root = PyObject(pynone) - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - new_root = mod._default_root - if !pyisnone(new_root) - root = new_root - end - if !pyisnone(root) - while pytruth(root.dooneevent(flag)) - end - end - end - else - error("invalid gui: $(repr(g))") - end - g => EVENT_LOOPS[g] -end diff --git a/src/old/import.jl b/src/old/import.jl deleted file mode 100644 index 06d0263d..00000000 --- a/src/old/import.jl +++ /dev/null @@ -1,34 +0,0 @@ -""" - pyimport(m, ...; [condapkg], [condachannel]) - pyimport(m => k, ...) - -Import and return the module `m`. - -If additionally `k` is given, then instead returns this attribute from `m`. If it is a tuple, a tuple of attributes is returned. - -If two or more arguments are given, they are all imported and returned as a tuple. - -If there is no such module and `condapkg` is given, then the given package will be automatically installed. Additionally, `condachannel` specifies the channel. -""" -function pyimport(m; condapkg::Union{Nothing,AbstractString}=nothing, condachannel::AbstractString="") - if condapkg === nothing || !CONFIG.isconda - m isa AbstractString ? check(C.PyImport_ImportModule(m)) : check(C.PyImport_Import(pyobject(m))) - else - try - pyimport(m) - catch err - if err isa PythonRuntimeError && pyissubclass(err.t, pymodulenotfounderror) - Conda.add(condapkg, CONFIG.condaenv; channel=condachannel) - pyimport(m) - else - rethrow() - end - end - end -end -function pyimport(x::Pair; opts...) - m = pyimport(x[1]; opts...) - x[2] isa Tuple ? map(k->pygetattr(m, k), x[2]) : pygetattr(m, x[2]) -end -pyimport(m1, m2, ms...) = (pyimport(m1), pyimport(m2), map(pyimport, ms)...) -export pyimport diff --git a/src/old/int.jl b/src/old/int.jl deleted file mode 100644 index c5f1af79..00000000 --- a/src/old/int.jl +++ /dev/null @@ -1,67 +0,0 @@ -const pyinttype = pylazyobject(() -> pybuiltins.int) -export pyinttype - -pyint(args...; opts...) = pyinttype(args...; opts...) -pyint(x::Integer) = - if typemin(Clonglong) ≤ x ≤ typemax(Clonglong) - check(C.PyLong_FromLongLong(x)) - else - # TODO: it's probably faster to do this in base 16 - check(C.PyLong_FromString(string(convert(BigInt, x)), C_NULL, 10)) - end -pyint(x::Unsigned) = - if x ≤ typemax(Culonglong) - check(C.PyLong_FromUnsignedLongLong(x)) - else - pyint(BigInt(x)) - end -export pyint - -pyisint(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_LONG_SUBCLASS) -export pyisint - -function pyint_tryconvert(::Type{T}, o::PyObject) where {T} - if BigInt <: T - # if it fits in a longlong, use that - rl = C.PyLong_AsLongLong(o) - if rl != -1 || !pyerroccurred() - # NOTE: In this case we return an Int if possible, since a lot of Julia functions take an Int but not a BigInt - # This is the only major exception to the rule that the Python type determines the Julia type. - return (Int <: T && typemin(Int) ≤ rl ≤ typemax(Int)) ? Int(rl) : BigInt(rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - else - pyerrclear() - return parse(BigInt, pystr(String, o)) - end - elseif (S = _typeintersect(T, Integer)) != Union{} - if S <: Unsigned - # if it fits in a ulonglong, use that - rl = C.PyLong_AsUnsignedLongLong(o) - if rl != zero(Culonglong)-one(Culonglong) || !pyerroccurred() - return tryconvert(S, rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - elseif S in (UInt8, UInt16, UInt32, UInt64, UInt128) && sizeof(S) ≤ sizeof(Culonglong) - pyerrclear() - return PyConvertFail() - end - else - # if it fits in a longlong, use that - rl = C.PyLong_AsLongLong(o) - if rl != -1 || !pyerroccurred() - return tryconvert(S, rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - elseif S in (Int8, Int16, Int32, Int64, Int128) && sizeof(S) ≤ sizeof(Clonglong) - pyerrclear() - return PyConvertFail() - end - end - # last resort: print to a string - pyerrclear() - return tryconvert(S, parse(BigInt, pystr(String, o))) - else - tryconvert(T, pyint_tryconvert(BigInt, o)) - end -end diff --git a/src/old/io.jl b/src/old/io.jl deleted file mode 100644 index ee1a6306..00000000 --- a/src/old/io.jl +++ /dev/null @@ -1,5 +0,0 @@ -const pyiounsupportedoperation = pylazyobject(() -> pyimport("io").UnsupportedOperation) - -pybufferedio(io::IO) = pyjlbufferedio(io) -pytextio(io::IO) = pyjltextio(io) -export pybufferedio, pytextio diff --git a/src/old/list.jl b/src/old/list.jl deleted file mode 100644 index affaa490..00000000 --- a/src/old/list.jl +++ /dev/null @@ -1,25 +0,0 @@ -const pylisttype = pylazyobject(() -> pybuiltins.list) -export pylisttype - -pyislist(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_LIST_SUBCLASS) -export pyislist - -pylist() = check(C.PyList_New(0)) -pylist(args...; opts...) = pylisttype(args...; opts...) -pylist(x::Union{Tuple,AbstractVector}) = pylist_fromiter(x) -export pylist - -function pylist_fromiter(xs) - r = pylist() - for x in xs - xo = pyobject(x) - check(C.PyList_Append(r, xo)) - end - return r -end - -pycollist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pycollist(y) for y in eachslice(x; dims=N)) -export pycollist - -pyrowlist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pyrowlist(y) for y in eachslice(x; dims=1)) -export pyrowlist diff --git a/src/old/none.jl b/src/old/none.jl deleted file mode 100644 index f5a70c38..00000000 --- a/src/old/none.jl +++ /dev/null @@ -1,12 +0,0 @@ -const pynone = pylazyobject(() -> pybuiltins.None) -export pynone - -const pynonetype = pylazyobject(() -> pytype(pynone)) -export pynonetype - -pyisnone(o::PyObject) = pyis(o, pynone) -export pyisnone - -function pynone_tryconvert(::Type{T}, o::PyObject) where {T} - tryconvert(T, nothing) -end diff --git a/src/old/number.jl b/src/old/number.jl deleted file mode 100644 index a919219e..00000000 --- a/src/old/number.jl +++ /dev/null @@ -1,41 +0,0 @@ -pyadd(o1, o2) = check(C.PyNumber_Add(pyobject(o1), pyobject(o2))) -pysub(o1, o2) = check(C.PyNumber_Subtract(pyobject(o1), pyobject(o2))) -pymul(o1, o2) = check(C.PyNumber_Multiply(pyobject(o1), pyobject(o2))) -pymatmul(o1, o2) = check(C.PyNumber_MatrixMultiply(pyobject(o1), pyobject(o2))) -pyfloordiv(o1, o2) = check(C.PyNumber_FloorDivide(pyobject(o1), pyobject(o2))) -pytruediv(o1, o2) = check(C.PyNumber_TrueDivide(pyobject(o1), pyobject(o2))) -pymod(o1, o2) = check(C.PyNumber_Remainder(pyobject(o1), pyobject(o2))) -pydivmod(o1, o2) = check(C.PyNumber_DivMod(pyobject(o1), pyobject(o2))) -pypow(o1, o2, o3=pynone) = check(C.PyNumber_Power(pyobject(o1), pyobject(o2), pyobject(o3))) -pyneg(o) = check(C.PyNumber_Negative(pyobject(o))) -pypos(o) = check(C.PyNumber_Positive(pyobject(o))) -pyabs(o) = check(C.PyNumber_Absolute(pyobject(o))) -pyinv(o) = check(C.PyNumber_Invert(pyobject(o))) -pylshift(o1, o2) = check(C.PyNumber_Lshift(pyobject(o1), pyobject(o2))) -pyrshift(o1, o2) = check(C.PyNumber_Rshift(pyobject(o1), pyobject(o2))) -pyand(o1, o2) = check(C.PyNumber_And(pyobject(o1), pyobject(o2))) -pyxor(o1, o2) = check(C.PyNumber_Xor(pyobject(o1), pyobject(o2))) -pyor(o1, o2) = check(C.PyNumber_Or(pyobject(o1), pyobject(o2))) -export pyadd, pysub, pymul, pymatmul, pyfloordiv, pytruediv, pymod, pydivmod, pypow, pyneg, pypos, pyabs, pyinv, pylshift, pyrshift, pyand, pyxor, pyor - -# in-place -pyiadd(o1, o2) = check(C.PyNumber_InPlaceAdd(pyobject(o1), pyobject(o2))) -pyisub(o1, o2) = check(C.PyNumber_InPlaceSubtract(pyobject(o1), pyobject(o2))) -pyimul(o1, o2) = check(C.PyNumber_InPlaceMultiply(pyobject(o1), pyobject(o2))) -pyimatmul(o1, o2) = check(C.PyNumber_InPlaceMatrixMultiply(pyobject(o1), pyobject(o2))) -pyifloordiv(o1, o2) = check(C.PyNumber_InPlaceFloorDivide(pyobject(o1), pyobject(o2))) -pyitruediv(o1, o2) = check(C.PyNumber_InPlaceTrueDivide(pyobject(o1), pyobject(o2))) -pyimod(o1, o2) = check(C.PyNumber_InPlaceRemainder(pyobject(o1), pyobject(o2))) -pyipow(o1, o2, o3=pynone) = check(C.PyNumber_InPlacePower(pyobject(o1), pyobject(o2), pyobject(o3))) -pyilshift(o1, o2) = check(C.PyNumber_InPlaceLshift(pyobject(o1), pyobject(o2))) -pyirshift(o1, o2) = check(C.PyNumber_InPlaceRshift(pyobject(o1), pyobject(o2))) -pyiand(o1, o2) = check(C.PyNumber_InPlaceAnd(pyobject(o1), pyobject(o2))) -pyixor(o1, o2) = check(C.PyNumber_InPlaceXor(pyobject(o1), pyobject(o2))) -pyior(o1, o2) = check(C.PyNumber_InPlaceOr(pyobject(o1), pyobject(o2))) -export pyiadd, pyisub, pyimul, pyimatmul, pyifloordiv, pyitruediv, pyimod, pyipow, pyilshift, pyirshift, pyiand, pyixor, pyior - -# conversions -pyint(o) = check(C.PyNumber_Long(pyobject(o))) -pyfloat(o) = check(C.PyNumber_Float(pyobject(o))) -pyindex(o) = check(C.PyNumber_Index(pyobject(o))) -export pyint, pyfloat, pyindex diff --git a/src/old/numpy.jl b/src/old/numpy.jl deleted file mode 100644 index c267ca68..00000000 --- a/src/old/numpy.jl +++ /dev/null @@ -1 +0,0 @@ -const pynumpy = pylazyobject(() -> pyimport("numpy")) diff --git a/src/old/object.jl b/src/old/object.jl deleted file mode 100644 index d5a803a9..00000000 --- a/src/old/object.jl +++ /dev/null @@ -1,212 +0,0 @@ -### PYOBJECT - -mutable struct PyObject - ptr :: CPyPtr - get :: Function - function PyObject(::Val{:new}, src::Union{Function,Ptr}, borrowed::Bool=false) - if src isa Ptr - o = new(CPyPtr(src)) - borrowed && C.Py_IncRef(src) - else - o = new(CPyPtr(C_NULL), src) - end - finalizer(o) do o - if CONFIG.isinitialized - ptr = getfield(o, :ptr) - if ptr != C_NULL - with_gil(false) do - C.Py_DecRef(ptr) - end - end - end - setfield!(o, :ptr, CPyPtr(C_NULL)) - nothing - end - o - end -end -export PyObject - -""" - pynewobject(ptr::Ptr) - -Construct a `PyObject` from the new reference `ptr`. -""" -pynewobject(ptr::Ptr) = PyObject(Val(:new), ptr, false) - -""" - pyborrowedobject(ptr::Ptr) - -Construct a `PyObject` from the borrowed reference `ptr`. Its refcount is incremented. -""" -pyborrowedobject(ptr::Ptr) = PyObject(Val(:new), ptr, true) - -""" - pylazyobject(f::Function) - -Construct a `PyObject` lazily whose value is equal to `f()`, except evaluation of `f` is deferred until the object is used at run-time. - -Use these to construct module-level constants, such as `pynone`. -""" -pylazyobject(f::Function) = PyObject(Val(:new), f) - -function pyptr(o::PyObject) - # check if assigned - ptr = getfield(o, :ptr) - ptr == C_NULL || return ptr - # otherwise generate a value - val = getfield(o, :get)() - ptr = pyptr(val) - setfield!(o, :ptr, ptr) - C.Py_IncRef(ptr) - return ptr -end - -PyObject(o::PyObject) = o -PyObject(o) = PyObject(pyobject(o)) - -Base.convert(::Type{PyObject}, o::PyObject) = o -Base.convert(::Type{PyObject}, o) = PyObject(o) - -pyincref!(o::PyObject) = (C.Py_IncRef(pyptr(o)); o) - -Base.unsafe_convert(::Type{CPyPtr}, o::PyObject) = pyptr(o) - -### CONVERSIONS - -const pyobjecttype = pylazyobject(() -> pybuiltins.object) -export pyobjecttype - -pyobject(o::PyObject) = o -pyobject(args...; opts...) = pyobjecttype(args...; opts...) -pyobject(o::Nothing) = pynone -pyobject(o::Missing) = pynone -pyobject(o::Tuple) = pytuple(o) -pyobject(o::Pair) = pytuple(o) -pyobject(o::AbstractString) = pystr(o) -pyobject(o::Bool) = pybool(o) -pyobject(o::Integer) = pyint(o) -pyobject(o::Union{Float16,Float32,Float64}) = pyfloat(o) -pyobject(o::Union{Complex{Float16}, Complex{Float32}, Complex{Float64}}) = pycomplex(o) -pyobject(o::Rational) = pyfraction(o) -pyobject(o::AbstractRange{<:Integer}) = pyrange(o) -pyobject(o::DateTime) = pydatetime(o) -pyobject(o::Date) = pydate(o) -pyobject(o::Time) = pytime(o) -pyobject(o::IO) = pybufferedio(o) -pyobject(o::Function) = pyjl(o, Function) # functions are singleton types - this avoids compiling a separate type for each one -pyobject(o) = pyjl(o) -export pyobject - -### ABSTRACT OBJECT API - -pyrefcnt(o::PyObject) = C.Py_RefCnt(o) -export pyrefcnt - -pyis(o1::PyObject, o2::PyObject) = pyptr(o1) == pyptr(o2) -export pyis - -pyhasattr(o::PyObject, k::AbstractString) = check(Bool, C.PyObject_HasAttrString(o, k)) -pyhasattr(o::PyObject, k::Symbol) = pyhasattr(o, String(k)) -pyhasattr(o::PyObject, k) = check(Bool, C.PyObject_HasAttr(o, pyobject(k))) -export pyhasattr - -pygetattr(o::PyObject, k::AbstractString) = check(C.PyObject_GetAttrString(o, k)) -pygetattr(o::PyObject, k::Symbol) = pygetattr(o, String(k)) -pygetattr(o::PyObject, k) = check(C.PyObject_GetAttr(o, pyobject(k))) -export pygetattr - -pysetattr(o::PyObject, k::AbstractString, v) = check(Nothing, C.PyObject_SetAttrString(o, k, pyobject(v))) -pysetattr(o::PyObject, k::Symbol, v) = pysetattr(o, String(k), v) -pysetattr(o::PyObject, k, v) = check(Nothing, C.PyObject_SetAttr(o, pyobject(k), pyobject(v))) -export pysetattr - -pydelattr(o::PyObject, k::AbstractString) = check(Nothing, C.PyObject_DelAttrString(o, k)) -pydelattr(o::PyObject, k::Symbol) = pydelattr(o, String(k)) -pydelattr(o::PyObject, k) = check(Nothing, C.PyObject_DelAttr(o, pyobject(k))) -export pydelattr - -pyrichcompare(::Type{PyObject}, o1, o2, op::Cint) = check(C.PyObject_RichCompare(pyobject(o1), pyobject(o2), op)) -pyrichcompare(::Type{Bool}, o1, o2, op::Cint) = check(Bool, C.PyObject_RichCompareBool(pyobject(o1), pyobject(o2), op)) -pyrichcompare(o1, o2, op) = pyrichcompare(PyObject, o1, o2, op) -pyrichcompare(::Type{T}, o1, o2, ::typeof(< )) where {T} = pyrichcompare(T, o1, o2, C.Py_LT) -pyrichcompare(::Type{T}, o1, o2, ::typeof(<=)) where {T} = pyrichcompare(T, o1, o2, C.Py_LE) -pyrichcompare(::Type{T}, o1, o2, ::typeof(==)) where {T} = pyrichcompare(T, o1, o2, C.Py_EQ) -pyrichcompare(::Type{T}, o1, o2, ::typeof(!=)) where {T} = pyrichcompare(T, o1, o2, C.Py_NE) -pyrichcompare(::Type{T}, o1, o2, ::typeof(> )) where {T} = pyrichcompare(T, o1, o2, C.Py_GT) -pyrichcompare(::Type{T}, o1, o2, ::typeof(>=)) where {T} = pyrichcompare(T, o1, o2, C.Py_GE) -export pyrichcompare - -pyeq(args...) = pyrichcompare(args..., ==) -pyne(args...) = pyrichcompare(args..., !=) -pylt(args...) = pyrichcompare(args..., < ) -pyle(args...) = pyrichcompare(args..., <=) -pygt(args...) = pyrichcompare(args..., > ) -pyge(args...) = pyrichcompare(args..., >=) -export pyeq, pyne, pylt, pyle, pygt, pyge - -pyrepr(o) = check(C.PyObject_Repr(pyobject(o))) -pyrepr(::Type{String}, o) = pystr_asjuliastring(pyrepr(o)) -export pyrepr - -pyascii(o) = check(C.PyObject_ASCII(pyobject(o))) -pyascii(::Type{String}, o) = pystr_asjuliastring(pyascii(o)) -export pyascii - -pystr(o) = check(C.PyObject_Str(pyobject(o))) -pystr(::Type{String}, o) = pystr_asjuliastring(pystr(o)) -export pystr - -pybytes(o) = check(C.PyObject_Bytes(pyobject(o))) -export pybytes - -pyissubclass(o1::PyObject, o2::PyObject) = check(Bool, C.PyObject_IsSubclass(o1, o2)) -export pyissubclass - -pyisinstance(o::PyObject, t::PyObject) = check(Bool, C.PyObject_IsInstance(o, t)) -export pyisinstance - -pyhash(o) = check(C.PyObject_Hash(pyobject(o))) -export pyhash - -pytruth(o) = check(Bool, C.PyObject_IsTrue(pyobject(o))) -export pytruth - -pylen(o) = check(C.PyObject_Length(pyobject(o))) -export pylen - -pygetitem(o::PyObject, k) = check(C.PyObject_GetItem(o, pyobject(k))) -export pygetitem - -pysetitem(o::PyObject, k, v) = check(Nothing, C.PyObject_SetItem(o, pyobject(k), pyobject(v))) -export pysetitem - -pydelitem(o::PyObject, k) = check(Nothing, C.PyObject_DelItem(o, pyobject(k))) -export pydelitem - -pydir(o::PyObject) = check(C.PyObject_Dir(o)) -export pydir - -pyiter(o) = check(C.PyObject_GetIter(pyobject(o))) -pyiter(args...; opts...) = pybuiltins.iter(args...; opts...) -export pyiter - -pycall(f, args...; kwargs...) = - if !isempty(kwargs) - argso = pytuple_fromiter(args) - kwargso = pydict_fromstringiter(kwargs) - check(C.PyObject_Call(f, argso, kwargso)) - elseif !isempty(args) - argso = pytuple_fromiter(args) - check(C.PyObject_CallObject(f, argso)) - else - check(C.PyObject_CallObject(f, C_NULL)) - end -export pycall - -### MODULE OBJECTS - -const pymoduletype = pylazyobject(() -> pytype(pybuiltins)) -export pymoduletype - -pyismodule(o::PyObject) = pyisinstance(o, pymoduletype) diff --git a/src/old/pandas.jl b/src/old/pandas.jl deleted file mode 100644 index 0657f7b4..00000000 --- a/src/old/pandas.jl +++ /dev/null @@ -1,110 +0,0 @@ -const pypandas = pylazyobject(() -> pyimport("pandas")) -const pypandasdataframetype = pylazyobject(() -> pypandas.DataFrame) - -asvector(x::AbstractVector) = x -asvector(x) = collect(x) - -""" - pycolumntable(src) - -Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. -""" -function pycolumntable(src) - cols = Tables.columns(src) - pydict_fromiter(pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) -end -pycolumntable(; cols...) = pycolumntable(cols) -export pycolumntable - -""" - pyrowtable(src) - -Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. -""" -function pyrowtable(src) - rows = Tables.rows(src) - names = Tables.columnnames(rows) - pynames = [pystr(String(n)) for n in names] - pylist_fromiter(pydict_fromiter(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) -end -pyrowtable(; cols...) = pyrowtable(cols) -export pyrowtable - -""" - pypandasdataframe([src]; ...) - -Construct a pandas dataframe from `src`. - -Usually equivalent to `pyimport("pandas").DataFrame(src, ...)`, but `src` may also be `Tables.jl`-compatible table. -""" -pypandasdataframe(t::PyObject; opts...) = pypandasdataframetype(t; opts...) -pypandasdataframe(; opts...) = pypandasdataframetype(; opts...) -function pypandasdataframe(t; opts...) - if Tables.istable(t) - cs = Tables.columns(t) - pypandasdataframetype(pydict_fromstringiter(string(c) => asvector(Tables.getcolumn(cs, c)) for c in Tables.columnnames(cs)); opts...) - else - pypandasdataframetype(t; opts...) - end -end -export pypandasdataframe - -multidict(src) = Dict(k=>v for (ks,v) in src for k in (ks isa Vector ? ks : [ks])) - -""" - PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) - -Wrap the Pandas dataframe `o` as a Julia table. - -This object satisfies the `Tables.jl` and `TableTraits.jl` interfaces. - -- `:indexname` is the name of the index column when converting this to a table, and may be `nothing` to exclude the index. -- `:columntypes` is an iterable of `columnname=>type` or `[columnnames...]=>type` pairs, used when converting to a table. -- `:copy` is true to copy columns on conversion. -""" -struct PyPandasDataFrame - o :: PyObject - indexname :: Union{Symbol, Nothing} - columntypes :: Dict{Symbol, Type} - copy :: Bool -end -PyPandasDataFrame(o::PyObject; indexname=:index, columntypes=(), copy=false) = PyPandasDataFrame(o, indexname, multidict(columntypes), copy) -export PyPandasDataFrame - -function pypandasdataframe_tryconvert(::Type{T}, o::PyObject) where {T} - if PyPandasDataFrame <: T - return PyPandasDataFrame(o) - else - tryconvert(T, PyPandasDataFrame(o)) - end -end - -Base.show(io::IO, x::PyPandasDataFrame) = print(io, x.o) - -### Tables.jl / TableTraits.jl integration - -Tables.istable(::Type{PyPandasDataFrame}) = true -Tables.columnaccess(::Type{PyPandasDataFrame}) = true -function Tables.columns(x::PyPandasDataFrame) - # columns - names = Symbol[Symbol(pystr(String, c)) for c in x.o.columns] - columns = PyVector[PyVector{get(x.columntypes, c, missing)}(x.o[pystr(c)].to_numpy()) for c in names] - # index - if x.indexname !== nothing - if x.indexname ∈ names - error("table already has column called $(x.indexname), cannot use it for index") - else - pushfirst!(names, x.indexname) - pushfirst!(columns, PyArray{get(x.columntypes, x.indexname, missing)}(x.o.index.to_numpy())) - end - end - if x.copy - columns = map(copy, columns) - end - return NamedTuple{Tuple(names)}(Tuple(columns)) -end - -IteratorInterfaceExtensions.isiterable(x::PyPandasDataFrame) = true -IteratorInterfaceExtensions.getiterator(x::PyPandasDataFrame) = IteratorInterfaceExtensions.getiterator(Tables.rows(x)) - -TableTraits.isiterabletable(x::PyPandasDataFrame) = true diff --git a/src/old/pywith.jl b/src/old/pywith.jl deleted file mode 100644 index ec62d326..00000000 --- a/src/old/pywith.jl +++ /dev/null @@ -1,28 +0,0 @@ -""" - pywith(f, o) - -Simulates the Python `with` statement, calling `f(x)` where `x=o.__enter__()`. -""" -function pywith(f, o::PyObject) - enter = pytype(o).__enter__ - exit = pytype(o).__exit__ - value = enter(o) - hit_except = false - try - return f(value) - catch err - if err isa PythonRuntimeError - hit_except = true - if pytruth(exit(o, err.t, err.v, err.b)) - rethrow() - end - else - rethrow() - end - finally - if !hit_except - exit(o, pynone, pynone, pynone) - end - end -end -export pywith diff --git a/src/old/range.jl b/src/old/range.jl deleted file mode 100644 index cc6ca11f..00000000 --- a/src/old/range.jl +++ /dev/null @@ -1,64 +0,0 @@ -const pyrangetype = pylazyobject(() -> pybuiltins.range) -export pyrangetype - -pyrange(args...; opts...) = pyrangetype(args...; opts...) -pyrange(x::AbstractRange{<:Integer}) = pyrange(first(x), last(x)+sign(step(x)), step(x)) -export pyrange - -pyisrange(o::PyObject) = pytypecheck(o, pyrangetype) -export pyisrange - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {A<:Integer, C<:Integer, T<:StepRange{A,C}} - a = pyint_tryconvert(A, o.start) - a === PyConvertFail() && return a - b = pyint_tryconvert(A, o.stop) - b === PyConvertFail() && return b - c = pyint_tryconvert(C, o.step) - c === PyConvertFail() && return c - return StepRange{A,C}(a, c, b-oftype(b, sign(c))) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {A<:Integer, T<:UnitRange{A}} - o.step == pyint(1) || return PyConvertFail() - a = pyint_tryconvert(A, o.start) - a === PyConvertFail() && return a - b = pyint_tryconvert(A, o.stop) - b === PyConvertFail() && return b - return UnitRange{A}(a, b-one(b)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:StepRange} - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:UnitRange} - tryconvert(T, pyrange_tryconvert(UnitRange{BigInt}, o)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {I<:Integer, T<:AbstractRange{I}} - if (S = _typeintersect(T, StepRange{I,I})) != Union{} - pyrange_tryconvert(S, o) - elseif (S = _typeintersect(T, UnitRange{I})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{I, I}, o)) - end -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:AbstractRange{<:Integer}} - if (S = _typeintersect(T, StepRange{<:Integer, <:Integer})) != Union{} - pyrange_tryconvert(S, o) - elseif (S = _typeintersect(T, UnitRange{<:Integer})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) - end -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, AbstractRange{<:Integer})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) - end -end diff --git a/src/old/sequence.jl b/src/old/sequence.jl deleted file mode 100644 index 53b6c537..00000000 --- a/src/old/sequence.jl +++ /dev/null @@ -1,2 +0,0 @@ -pycontains(o, v) = check(Bool, C.PySequence_Contains(pyobject(o), pyobject(v))) -export pycontains diff --git a/src/old/set.jl b/src/old/set.jl deleted file mode 100644 index 863e0083..00000000 --- a/src/old/set.jl +++ /dev/null @@ -1,14 +0,0 @@ -const pysettype = pylazyobject(() -> pybuiltins.set) -const pyfrozensettype = pylazyobject(() -> pybuiltins.frozenset) -export pysettype, pyfrozensettype - -pyisset(o::PyObject) = pytypecheck(o, pysettype) -pyisfrozenset(o::PyObject) = pytypecheck(o, pyfrozensettype) -pyisanyset(o::PyObject) = pyisset(o) || pyisfrozenset(o) -export pyisset, pyisfrozenset, pyisanyset - -pyset(args...; opts...) = pysettype(args...; opts...) -pyset(x::Union{Tuple,AbstractVector}) = pyset(pylist(x)) -pyfrozenset(args...; opts...) = pyfrozensettype(args...; opts...) -pyfrozenset(x::Union{Tuple,AbstractVector}) = pyfrozenset(pylist(x)) -export pyset, pyfrozenset diff --git a/src/old/slice.jl b/src/old/slice.jl deleted file mode 100644 index 196a4840..00000000 --- a/src/old/slice.jl +++ /dev/null @@ -1,8 +0,0 @@ -const pyslicetype = pylazyobject(() -> pybuiltins.slice) -export pyslicetype - -pyslice(args...; opts...) = pyslicetype(args...; opts...) -export pyslice - -pyisslice(o::PyObject) = pytypecheck(o, pyslicetype) -export pyisslice diff --git a/src/old/stdlib.jl b/src/old/stdlib.jl deleted file mode 100644 index 3fbf01ef..00000000 --- a/src/old/stdlib.jl +++ /dev/null @@ -1,13 +0,0 @@ -for m in ["os", "sys", "pprint", "traceback", "numbers", "math", "collections", "collections.abc", "datetime", "fractions", "io", "types", "pdb"] - j = Symbol(:py, replace(m, '.'=>""), :module) - @eval const $j = pylazyobject(() -> pyimport($m)) -end - -""" - pynewclass(name, bases=(), kwargs=nothing; attrs...) - -Create a new Python class with the given name, base classes and attributes. -""" -pynewclass(name, bases=(), kwds=nothing; attrs...) = - pytypesmodule.new_class(name, bases, kwds, pyjlfunction(ns -> (for (k,v) in attrs; ns[string(k)] = v; end))) -export pynewclass diff --git a/src/old/str.jl b/src/old/str.jl deleted file mode 100644 index f42bbaeb..00000000 --- a/src/old/str.jl +++ /dev/null @@ -1,33 +0,0 @@ -const pystrtype = pylazyobject(() -> pybuiltins.str) -export pystrtype - -pystr(x::Union{String,SubString{String}}) = check(C.PyUnicode_DecodeUTF8(x, ncodeunits(x), C_NULL)) -pystr(x::AbstractString) = pystr(convert(String, x)) -pystr(x::AbstractChar) = pystr(String(x)) -pystr(x::Symbol) = pystr(String(x)) -pystr(args...; opts...) = pystrtype(args...; opts...) -export pystr - -pystr_asutf8string(o::PyObject) = check(C.PyUnicode_AsUTF8String(o)) - -pystr_asjuliastring(o::PyObject) = pybytes_asjuliastring(pystr_asutf8string(o)) - -pyisstr(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_UNICODE_SUBCLASS) -export pyisstr - -function pystr_tryconvert(::Type{T}, o::PyObject) where {T} - x = pystr_asjuliastring(o) - if (S = _typeintersect(T, String)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, AbstractString)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, Symbol)) != Union{} - convert(S, Symbol(x)) - elseif (S = _typeintersect(T, Char)) != Union{} - length(x)==1 ? convert(S, x[1]) : PyConvertFail() - elseif (S = _typeintersect(T, AbstractChar)) != Union{} - length(x)==1 ? convert(S, x[1]) : PyConvertFail() - else - tryconvert(T, x) - end -end diff --git a/src/old/tuple.jl b/src/old/tuple.jl deleted file mode 100644 index 54ef6b44..00000000 --- a/src/old/tuple.jl +++ /dev/null @@ -1,28 +0,0 @@ -const pytupletype = pylazyobject(() -> pybuiltins.tuple) -export pytupletype - -pyistuple(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_TUPLE_SUBCLASS) -export pyistuple - -pytuple() = check(C.PyTuple_New(0)) -pytuple(args...; opts...) = pytupletype(args...; opts...) -pytuple(x::Union{Tuple,AbstractVector,Pair}) = pytuple_fromiter(x) -export pytuple - -function pytuple_fromiter(xs) - n = length(xs) - t = check(C.PyTuple_New(n)) - for (i,x) in enumerate(xs) - xo = pyobject(x) - check(C.PyTuple_SetItem(t, i-1, pyincref!(xo))) - end - return t -end - -function pytuple_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, Tuple)) != Union{} - pyiterable_tryconvert(S, o) - else - PyConvertFail() - end -end diff --git a/src/old/type.jl b/src/old/type.jl deleted file mode 100644 index 9496242e..00000000 --- a/src/old/type.jl +++ /dev/null @@ -1,12 +0,0 @@ -const pytypetype = pylazyobject(() -> pybuiltins.type) -export pytypetype - -pyistype(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_TYPE_SUBCLASS) -export pyistype - -pytype(o::PyObject) = pyborrowedobject(C.Py_Type(o)) -export pytype - -pytypecheck(o::PyObject, t::PyObject) = !iszero(C.Py_TypeCheck(o, t)) - -pytypecheckfast(o::PyObject, f::Integer) = !iszero(C.Py_TypeCheckFast(o, f)) From f64e0ff26bbd58c0dd6d396dbd593d9b51fac66f Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Mon, 21 Dec 2020 19:30:28 +0000 Subject: [PATCH 10/14] dev --- juliapy/julia/__init__.py | 13 ++- src/{old => }/PyBuffer.jl | 7 +- src/PyException.jl | 73 +++++++------ src/PyRef.jl | 2 +- src/Python.jl | 3 + src/builtins.jl | 101 ++++++++++++++++-- src/cpython/CPython.jl | 10 ++ src/cpython/arg.jl | 109 ++++++++++++++++++++ src/cpython/bytes.jl | 25 +++-- src/cpython/dict.jl | 55 ++++++---- src/cpython/fraction.jl | 31 ++++++ src/cpython/juliabase.jl | 98 ++++++++++++++++++ src/cpython/juliaerror.jl | 37 +++++++ src/cpython/juliaraw.jl | 208 ++++++++++++++++++++++++++++++++++++++ src/cpython/list.jl | 28 +++++ src/cpython/newtype.jl | 125 +++++++++++++++++++++++ src/cpython/object.jl | 9 ++ src/cpython/set.jl | 52 ++++++++++ src/cpython/slice.jl | 17 ++++ src/cpython/str.jl | 45 +++++---- src/cpython/tuple.jl | 19 +++- src/init.jl | 171 ++++++++++++++++--------------- src/julia.jl | 28 +++++ src/matplotlib.jl | 53 ++++++++++ src/old/matplotlib.jl | 39 ------- src/old/newtype.jl | 79 --------------- 26 files changed, 1148 insertions(+), 289 deletions(-) rename src/{old => }/PyBuffer.jl (93%) create mode 100644 src/cpython/arg.jl create mode 100644 src/cpython/fraction.jl create mode 100644 src/cpython/juliabase.jl create mode 100644 src/cpython/juliaerror.jl create mode 100644 src/cpython/juliaraw.jl create mode 100644 src/cpython/newtype.jl create mode 100644 src/cpython/slice.jl create mode 100644 src/julia.jl create mode 100644 src/matplotlib.jl delete mode 100644 src/old/matplotlib.jl delete mode 100644 src/old/newtype.jl diff --git a/juliapy/julia/__init__.py b/juliapy/julia/__init__.py index f1503bf7..d8a10217 100644 --- a/juliapy/julia/__init__.py +++ b/juliapy/julia/__init__.py @@ -33,10 +33,15 @@ def _init_(): lib.jl_eval_string.restype = c.c_void_p res = lib.jl_eval_string( ''' - ENV["PYTHONJL_LIBPTR"] = "{}" - import Python - Python.with_gil() do - Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main) + try + ENV["PYTHONJL_LIBPTR"] = "{}" + import Python + Python.with_gil() do + Python.pyimport("sys").modules["julia"].Main = Python.pyjlraw(Main) + end + catch err + @error "Error loading Python.jl" err=err + rethrow() end '''.format(c.pythonapi._handle).encode('utf8')) if res is None: diff --git a/src/old/PyBuffer.jl b/src/PyBuffer.jl similarity index 93% rename from src/old/PyBuffer.jl rename to src/PyBuffer.jl index 68dbb11b..b52a57bc 100644 --- a/src/old/PyBuffer.jl +++ b/src/PyBuffer.jl @@ -1,5 +1,5 @@ """ - PyBuffer(o) + PyBuffer(o, [flags=C.PyBUF_FULL_RO]) A reference to the underlying buffer of `o`, if it satisfies the buffer protocol. @@ -20,16 +20,15 @@ Has the following properties: """ mutable struct PyBuffer info :: Array{C.Py_buffer, 0} - function PyBuffer(o::PyObject, flags::Integer=C.PyBUF_FULL_RO) + function PyBuffer(o, flags::Integer=C.PyBUF_FULL_RO) info = fill(C.Py_buffer()) check(C.PyObject_GetBuffer(o, pointer(info), flags)) b = new(info) finalizer(b) do b if CONFIG.isinitialized - err = with_gil(false) do + with_gil(false) do C.PyBuffer_Release(pointer(b.info)) end - check(err) end end b diff --git a/src/PyException.jl b/src/PyException.jl index 7d0ffa75..6a7075bb 100644 --- a/src/PyException.jl +++ b/src/PyException.jl @@ -7,7 +7,7 @@ mutable struct PyException <: Exception end export PyException -pythrow() = throw(PyException(Val(:new), C.PyErr_FetchTuple()...)) +pythrow() = throw(PyException(Val(:new), C.PyErr_FetchTuple(true)...)) """ check(x) @@ -81,36 +81,47 @@ function Base.showerror(io::IO, e::PyException) end end - # # if this is a Julia exception then recursively print it and its stacktrace - # if pyerrmatches(e.t, pyjlexception) - # try - # jp = pyjlgetvalue(e.v.args[0]) - # if jp !== nothing - # je, jb = jp - # print(io, "Python: Julia: ") - # showerror(io, je) - # if jb === nothing - # println(io) - # print(io, "Stacktrace: none") - # else - # io2 = IOBuffer() - # Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) - # printstyled(io, String(take!(io2))) - # end - # end - # if pyisnone(e.b) - # println(io) - # printstyled(io, "Python stacktrace: none") - # return - # else - # @goto pystacktrace - # end - # catch err - # println(io, "") - # end - # end - - # otherwise, print the Python exception ****** TODO ****** + # if this is a Julia exception then recursively print it and its stacktrace + if C.PyErr_GivenExceptionMatches(e.tref, C.PyExc_JuliaError()) != 0 + try + # Extract error value + vo = @pyv `$(e.vref).args[0]`::Any + if vo isa Tuple{Exception, Any} + je, jb = vo + else + je = vo + jb = nothing + end + print(io, "Python: Julia: ") + # Show exception + if je isa Exception + showerror(io, je) + else + print(io, je) + end + # Show backtrace + if jb === nothing + println(io) + print(io, "Stacktrace: none") + else + io2 = IOBuffer() + Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) + printstyled(io, String(take!(io2))) + end + # Show Python backtrace + if isnull(e.bref) + println(io) + printstyled(io, "Python stacktrace: none") + return + else + @goto pystacktrace + end + catch err + println("") + end + end + + # otherwise, print the Python exception print(io, "Python: ") # print the type name diff --git a/src/PyRef.jl b/src/PyRef.jl index 2bcddec4..5d59b55c 100644 --- a/src/PyRef.jl +++ b/src/PyRef.jl @@ -14,7 +14,7 @@ mutable struct PyRef x = new(CPyPtr(ptr)) borrowed && C.Py_IncRef(ptr) finalizer(x) do x - if !CONFIG.isinitialized + if CONFIG.isinitialized ptr = x.ptr if !isnull(ptr) with_gil(false) do diff --git a/src/Python.jl b/src/Python.jl index e108064b..8405f6e2 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -93,9 +93,12 @@ include("PyList.jl") include("PySet.jl") include("PyIterable.jl") include("PyIO.jl") +include("PyBuffer.jl") include("PyPandasDataFrame.jl") +include("julia.jl") include("gui.jl") +include("matplotlib.jl") include("init.jl") diff --git a/src/builtins.jl b/src/builtins.jl index 8bb457ff..1aefba10 100644 --- a/src/builtins.jl +++ b/src/builtins.jl @@ -640,12 +640,6 @@ pyinv(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Invert, x) pyinv(x) = pyinv(typeof(x), x) export pyinv -# pycollist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pycollist(y) for y in eachslice(x; dims=N)) -# export pycollist - -# pyrowlist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pyrowlist(y) for y in eachslice(x; dims=1)) -# export pyrowlist - """ pyiter([T=PyObject] x) :: T @@ -688,6 +682,101 @@ function pywith(f, _o, d=nothing) end export pywith +""" + pytuple([T=PyObject,] [x]) :: T + +Create a Python `tuple` from the elements of `x`. +""" +pytuple(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyTuple_Type(), x) : C.PyTuple_FromIter(x)) +pytuple(::Type{T}) where {T} = checknullconvert(T, C.PyTuple_New(0)) +pytuple(x) = pytuple(PyObject, x) +pytuple() = pytuple(PyObject) +export pytuple + +""" + pylist([T=PyObject,] [x]) :: T + +Create a Python `list` from the elements of `x`. +""" +pylist(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyList_Type(), x) : C.PyList_FromIter(x)) +pylist(::Type{T}) where {T} = checknullconvert(T, C.PyList_New(0)) +pylist(x) = pylist(PyObject, x) +pylist() = pylist(PyObject) +export pylist + +""" + pycollist([T=PyObject,] x::AbstractArray) :: T + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of columns. +""" +pycollist(::Type{T}, x::AbstractArray) where {T} = ndims(x)==0 ? pyconvert(T, x[]) : pylist(T, pycollist(PyRef, y) for y in eachslice(x; dims=ndims(x))) +pycollist(x::AbstractArray) = pycollist(PyObject, x) +export pycollist + +""" + pyrowlist([T=PyObject,] x::AbstractArray) :: T + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of rows. +""" +pyrowlist(::Type{T}, x::AbstractArray) where {T} = ndims(x)==0 ? pyconvert(T, x[]) : pylist(T, pyrowlist(PyRef, y) for y in eachslice(x; dims=1)) +pyrowlist(x::AbstractArray) = pyrowlist(PyObject, x) +export pyrowlist + +""" + pyset([T=PyObject,] [x]) :: T + +Create a Python `set` from the elements of `x`. +""" +pyset(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PySet_Type(), x) : C.PySet_FromIter(x)) +pyset(::Type{T}) where {T} = checknullconvert(T, C.PySet_New(C_NULL)) +pyset(x) = pyset(PyObject, x) +pyset() = pyset(PyObject) +export pyset + +""" + pyfrozenset([T=PyObject,] [x]) :: T + +Create a Python `frozenset` from the elements of `x`. +""" +pyfrozenset(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyFrozenSet_Type(), x) : C.PyFrozenSet_FromIter(x)) +pyfrozenset(::Type{T}) where {T} = checknullconvert(T, C.PyFrozenSet_New(C_NULL)) +pyfrozenset(x) = pyfrozenset(PyObject, x) +pyfrozenset() = pyfrozenset(PyObject) +export pyfrozenset + +""" + pydict([T=PyObject,] [x]) :: T + +Create a Python `dict` from the key-value pairs in `x`. +""" +pydict(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyDict_Type(), x) : C.PyDict_FromPairs(x)) +pydict(::Type{T}) where {T} = checknullconvert(T, C.PyDict_New()) +pydict(x) = pydict(PyObject, x) +pydict() = pydict(PyObject) +export pydict + +""" + pyslice([T=PyObject,] [start,] stop, [step]) :: T + +Equivalent to `slice(start, stop, step)` in Python (or `start:stop:step` while indexing). +""" +pyslice(::Type{T}, x) where {T} = cpyop(T, x->C.PySlice_New(C_NULL, x, C_NULL), x) +pyslice(::Type{T}, x, y) where {T} = cpyop(T, (x,y)->C.PySlice_New(x, y, C_NULL), x, y) +pyslice(::Type{T}, x, y, z) where {T} = cpyop(T, C.PySlice_New, x, y, z) +pyslice(x) = pyslice(PyObject, x) +pyslice(x, y) = pyslice(PyObject, x, y) +pyslice(x, y, z) = pyslice(PyObject, x, y, z) +export pyslice + +""" + pyellipsis([T=PyObject]) + +Equivalent to `Ellipsis` in Python (or `...` while indexing). +""" +pyellipsis(::Type{T}) where {T} = checknullconvert(T, C.PyEllipsis_New()) +pyellipsis() = pyellipsis(PyObject) +export pyellipsis + ### MULTIMEDIA DISPLAY for (mime, method) in [ diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index 06eb03e4..514fbb1d 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -50,6 +50,13 @@ include("collections.jl") include("range.jl") include("ctypes.jl") include("numpy.jl") +include("slice.jl") +include("fraction.jl") +include("newtype.jl") +include("juliaerror.jl") +include("juliabase.jl") +include("juliaraw.jl") +include("arg.jl") __init__() = begin PyObject_TryConvert_AddRules("builtins.NoneType", [ @@ -124,6 +131,9 @@ __init__() = begin PyObject_TryConvert_AddRules("collections.abc.Mapping", [ (Dict, PyMapping_ConvertRule_dict), ]) + PyObject_TryConvert_AddRules("julia.ValueBase", [ + (Any, PyJuliaValue_TryConvert_any), + ]) PyObject_TryConvert_AddExtraTypes([ PyIterableABC_Type, PyCallableABC_Type, diff --git a/src/cpython/arg.jl b/src/cpython/arg.jl new file mode 100644 index 00000000..cb506ba6 --- /dev/null +++ b/src/cpython/arg.jl @@ -0,0 +1,109 @@ +PyArg_CheckNumArgsEq(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m == n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsGe(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m ≥ n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes at least $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsLe(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m ≤ n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes at most $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsBetween(name::String, args::PyPtr, n0::Integer, n1::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if n0 ≤ m ≤ n1 + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes $n0 to $n1 arguments ($m given)") + return -1 + end +end + +PyArg_CheckNoKwargs(name::String, kwargs::PyPtr) = begin + if isnull(kwargs) || PyObject_Length(kwargs) == 0 + return 0 + else + it = PyObject_GetIter(kwargs) + isnull(it) && return -1 + argnames = String[] + while true + argnameo = PyIter_Next(it) + if !isnull(argnameo) + argname = PyUnicode_AsString(argnameo) + Py_DecRef(argnameo) + isempty(argname) && PyErr_IsSet() && (Py_DecRef(it); return -1) + push!(argnames, argname) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + break + end + end + PyErr_SetString(PyExc_TypeError(), "$name() got unexpected keyword arguments: $(join(["'$n'" for n in argnames], ", "))") + return -1 + end +end + +struct NODEFAULT end + +PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}) = + if i !== nothing && !isnull(args) && 0 ≤ i ≤ PyTuple_Size(args) + return PyTuple_GetItem(args, i) + elseif k !== nothing && !isnull(kwargs) && (ro = PyDict_GetItemString(kwargs, k)) != PyPtr() + return ro + else + return PyPtr() + end + +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Union{Int,Nothing}=nothing, k::Union{String,Nothing}=nothing, d::Union{PyPtr,NODEFAULT}=NODEFAULT()) where {T} = begin + ro = PyArg_Find(args, kwargs, i, k) + if isnull(ro) + if k !== nothing + PyErr_SetString(PyExc_TypeError(), "$name() did not get required argument '$k'") + elseif i !== nothing + PyErr_SetString(PyExc_TypeError(), "$name() takes at least $(i+1) arguments (got $(isnull(args) ? 0 : PyTuple_Size(args)))") + else + error("impossible to satisfy this argument") + end + return -1 + end + r = PyObject_TryConvert(ro, T) + if r == -1 + return -1 + elseif r == 0 + if d === NODEFAULT() + PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'") + return -1 + else + putresult(T, d) + return 0 + end + else + return 0 + end +end +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}=nothing, d::Union{PyPtr,NODEFAULT}=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, args, PyPtr(), i, k, d) +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr, k::Union{String,Nothing}, d::Union{PyPtr, NODEFAULT}=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, args, kwargs, nothing, k, d) diff --git a/src/cpython/bytes.jl b/src/cpython/bytes.jl index 6cb1356c..d5308949 100644 --- a/src/cpython/bytes.jl +++ b/src/cpython/bytes.jl @@ -10,20 +10,31 @@ PyBytes_CheckExact(o) = Py_TypeCheckExact(o, PyBytes_Type()) PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = PyBytes_FromStringAndSize(pointer(s), sizeof(s)) -PyBytes_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T,X} = begin +PyBytes_AsString(o) = begin ptr = Ref{Ptr{Cchar}}() len = Ref{Py_ssize_t}() err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return -1 - v = copy(Base.unsafe_wrap(Vector{X}, Ptr{X}(ptr[]), len[])) - return putresult(T, v) + ism1(err) && return "" + Base.unsafe_string(ptr[], len[]) end -PyBytes_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin +PyBytes_AsVector(o, ::Type{T}=UInt8) where {T} = begin + T in (Int8, UInt8) || throw(MethodError(PyBytes_AsVector, (o, T))) ptr = Ref{Ptr{Cchar}}() len = Ref{Py_ssize_t}() err = PyBytes_AsStringAndSize(o, ptr, len) - ism1(err) && return -1 - v = Base.unsafe_string(ptr[], len[]) + ism1(err) && return T[] + copy(Base.unsafe_wrap(Vector{T}, Ptr{T}(ptr[]), len[])) +end + +PyBytes_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T,X} = begin + v = PyBytes_AsVector(o, X) + isempty(v) && PyErr_IsSet() && return -1 + return putresult(T, v) +end + +PyBytes_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin + v = PyBytes_AsString(o) + isempty(v) && PyErr_IsSet() && return -1 return putresult(T, v) end diff --git a/src/cpython/dict.jl b/src/cpython/dict.jl index 2e1c0d14..3cee4a00 100644 --- a/src/cpython/dict.jl +++ b/src/cpython/dict.jl @@ -1,4 +1,6 @@ @cdef :PyDict_New PyPtr () +@cdef :PyDict_GetItem PyPtr (PyPtr, PyPtr) +@cdef :PyDict_GetItemString PyPtr (PyPtr, Cstring) @cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) @cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) @cdef :PyDict_DelItemString Cint (PyPtr, Cstring) @@ -7,40 +9,53 @@ const PyDict_Type__ref = Ref(PyPtr()) PyDict_Type() = pyglobal(PyDict_Type__ref, :PyDict_Type) PyDict_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_DICT_SUBCLASS) + PyDict_CheckExact(o) = Py_TypeCheckExact(o, PyDict_Type()) PyDict_FromPairs(kvs) = begin r = PyDict_New() isnull(r) && return PyPtr() - for (k,v) in kvs - ko = PyObject_From(k) - isnull(ko) && (Py_DecRef(r); return PyPtr()) - vo = PyObject_From(v) - isnull(vo) && (Py_DecRef(r); Py_DecRef(ko); return PyPtr()) - err = PyDict_SetItem(r, ko, vo) - Py_DecRef(ko) - Py_DecRef(vo) - ism1(err) && (Py_DecRef(r); return PyPtr()) + try + for (k,v) in kvs + ko = PyObject_From(k) + isnull(ko) && (Py_DecRef(r); return PyPtr()) + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); Py_DecRef(ko); return PyPtr()) + err = PyDict_SetItem(r, ko, vo) + Py_DecRef(ko) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() end - r end PyDict_FromStringPairs(kvs) = begin r = PyDict_New() isnull(r) && return PyPtr() - for (k,v) in kvs - vo = PyObject_From(v) - isnull(vo) && (Py_DecRef(r); return PyPtr()) - err = PyDict_SetItemString(r, string(k), vo) - Py_DecRef(vo) - ism1(err) && (Py_DecRef(r); return PyPtr()) + try + for (k,v) in kvs + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); return PyPtr()) + err = PyDict_SetItemString(r, string(k), vo) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() end - r end -PyDict_From(x::Dict) = PyDict_FromPairs(x) -PyDict_From(x::Dict{String}) = PyDict_FromStringPairs(x) -PyDict_From(x::Dict{Symbol}) = PyDict_FromStringPairs(x) +PyDict_From(x::AbstractDict) = PyDict_FromPairs(x) +PyDict_From(x::AbstractDict{String}) = PyDict_FromStringPairs(x) +PyDict_From(x::AbstractDict{Symbol}) = PyDict_FromStringPairs(x) PyDict_From(x::NamedTuple) = PyDict_FromStringPairs(pairs(x)) PyDict_From(x::Base.Iterators.Pairs) = PyDict_FromPairs(x) PyDict_From(x::Base.Iterators.Pairs{String}) = PyDict_FromStringPairs(x) diff --git a/src/cpython/fraction.jl b/src/cpython/fraction.jl new file mode 100644 index 00000000..469ec99d --- /dev/null +++ b/src/cpython/fraction.jl @@ -0,0 +1,31 @@ +const PyFraction_Type__ref = Ref(PyPtr()) +PyFraction_Type() = begin + ptr = PyFraction_Type__ref[] + if isnull(ptr) + m = PyImport_ImportModule("fractions") + isnull(m) && return ptr + ptr = PyObject_GetAttrString(m, "Fraction") + Py_DecRef(m) + isnull(m) && return ptr + PyFraction_Type__ref[] = ptr + end + ptr +end + +PyFraction_From(x::Union{Rational,Integer}) = begin + t = PyFraction_Type() + isnull(t) && return PyPtr() + a = PyTuple_New(2) + isnull(a) && return PyPtr() + b = PyLong_From(numerator(x)) + isnull(b) && (Py_DecRef(a); return PyPtr()) + err = PyTuple_SetItem(a, 0, b) + ism1(err) && (Py_DecRef(a); return PyPtr()) + b = PyLong_From(denominator(x)) + isnull(b) && (Py_DecRef(a); return PyPtr()) + err = PyTuple_SetItem(a, 1, b) + ism1(err) && (Py_DecRef(a); return PyPtr()) + r = PyObject_CallObject(t, a) + Py_DecRef(a) + return r +end diff --git a/src/cpython/juliabase.jl b/src/cpython/juliabase.jl new file mode 100644 index 00000000..401aabc4 --- /dev/null +++ b/src/cpython/juliabase.jl @@ -0,0 +1,98 @@ +const PYJLGCCACHE = Dict{PyPtr, Any}() + +@kwdef struct PyJuliaValueObject + ob_base :: PyObject = PyObject() + value :: Ptr{Cvoid} = C_NULL + weaklist :: PyPtr = C_NULL +end + +pyjlbase_new(t::PyPtr, ::PyPtr, ::PyPtr) = begin + o = ccall(UnsafePtr{PyTypeObject}(t).alloc[!], PyPtr, (PyPtr, Py_ssize_t), t, 0) + if !isnull(o) + PyJuliaValue_SetValue(o, nothing) + UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C_NULL + end + o +end + +pyjlbase_init(o::PyPtr, args::PyPtr, kwargs::PyPtr) = begin + ism1(PyArg_CheckNumArgsEq("__init__", args, 1)) && return Cint(-1) + ism1(PyArg_CheckNoKwargs("__init__", kwargs)) && return Cint(-1) + ism1(PyArg_GetArg(Any, "__init__", args, kwargs, 0)) && return Cint(-1) + PyJuliaValue_SetValue(o, takeresult(Any)) + Cint(0) +end + +pyjlbase_dealloc(o::PyPtr) = begin + delete!(PYJLGCCACHE, o) + isnull(UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) || PyObject_ClearWeakRefs(o) + ccall(UnsafePtr{PyTypeObject}(Py_Type(o)).free[!], Cvoid, (PyPtr,), o) + nothing +end + +const PyJuliaBaseValue_Type__ref = Ref(PyPtr()) +PyJuliaBaseValue_Type() = begin + ptr = PyJuliaBaseValue_Type__ref[] + if isnull(ptr) + c = [] + t = fill(PyType_Create(c, + name = "julia.ValueBase", + basicsize = sizeof(PyJuliaValueObject), + new = pyglobal(:PyType_GenericNew), + init = @cfunction(pyjlbase_init, Cint, (PyPtr, PyPtr, PyPtr)), + dealloc = @cfunction(pyjlbase_dealloc, Cvoid, (PyPtr,)), + flags = Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), + weaklistoffset = fieldoffset(PyJuliaValueObject, 3), + getattro = pyglobal(:PyObject_GenericGetAttr), + setattro = pyglobal(:PyObject_GenericSetAttr), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaBaseValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaValue_Check(o) = Py_TypeCheck(o, PyJuliaBaseValue_Type()) != 0 + +PyJuliaValue_GetValue(__o) = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + p = UnsafePtr{PyJuliaValueObject}(o) + Base.unsafe_pointer_to_objref(p.value[!]) + end +end + +PyJuliaValue_SetValue(__o, v) = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + p = UnsafePtr{PyJuliaValueObject}(o) + p.value[!], PYJLGCCACHE[o] = Python.pointer_from_obj(v) + end +end + +PyJuliaValue_New(t, v) = begin + bt = PyJuliaBaseValue_Type() + isnull(bt) && return PyPtr() + PyType_IsSubtype(t, bt) != 0 || (PyErr_SetString(PyExc_TypeError(), "Expecting a subtype of 'julia.ValueBase'"); return PyPtr()) + o = _PyObject_New(t) + isnull(o) && return PyPtr() + UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C_NULL + PyJuliaValue_SetValue(o, v) + o +end + +PyJuliaBaseValue_New(v) = begin + t = PyJuliaBaseValue_Type() + isnull(t) && return PyPtr() + PyJuliaValue_New(t, v) +end + +PyJuliaValue_TryConvert_any(o, ::Type{T}, ::Type{S}) where {T,S} = begin + x = PyJuliaValue_GetValue(o) + putresult(T, tryconvert(S, x)) +end diff --git a/src/cpython/juliaerror.jl b/src/cpython/juliaerror.jl new file mode 100644 index 00000000..35f233dc --- /dev/null +++ b/src/cpython/juliaerror.jl @@ -0,0 +1,37 @@ +const PyExc_JuliaError__ref = Ref(PyPtr()) +PyExc_JuliaError() = begin + ptr = PyExc_JuliaError__ref[] + if isnull(ptr) + c = [] + t = fill(PyType_Create(c, + name = "julia.Error", + base = PyExc_BaseException(), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyExc_JuliaError__ref[] = ptr + end + ptr +end + +PyErr_SetJuliaError(err, bt=nothing) = begin + t = PyExc_JuliaError() + if isnull(t) + PyErr_Clear() + PyErr_SetString(PyExc_Exception(), "Julia error: $err (an error occurred while setting this error)") + else + if bt === nothing + bt = catch_backtrace() + end + v = PyJuliaBaseValue_New((err, bt)) + if isnull(v) + PyErr_Clear() + PyErr_SetString(t, string(err)) + else + PyErr_SetObject(t, v) + Py_DecRef(v) + end + end +end diff --git a/src/cpython/juliaraw.jl b/src/cpython/juliaraw.jl new file mode 100644 index 00000000..d092340b --- /dev/null +++ b/src/cpython/juliaraw.jl @@ -0,0 +1,208 @@ +JLRAWTYPES = Dict{Type, PyPtr}() + +const PyJuliaRawValue_Type__ref = Ref(PyPtr()) + +PyJuliaRawValue_Type() = begin + ptr = PyJuliaRawValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaBaseValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.RawValue", + base = base, + repr = pyjlraw_repr, + str = pyjlraw_str, + getattro = pyjlraw_getattro, + setattro = pyjlraw_setattro, + call = pyjlraw_call, + as_mapping = ( + length = pyjlraw_length, + subscript = pyjlraw_getitem, + ass_subscript = pyjlraw_setitem, + ), + methods = [ + (name="__dir__", flags=Py_METH_NOARGS, meth=pyjlraw_dir), + ] + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaRawValue_Type__ref[] = ptr + end + ptr +end + +pyjlraw_repr(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = "" + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err, catch_backtrace()) + return PyPtr() +end + +pyjlraw_str(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = string(x) + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err, catch_backtrace()) + return PyPtr() +end + +pyjlraw_getattro(xo::PyPtr, ko::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericGetAttr(xo, ko) + if isnull(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to get the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return PyPtr() + try + v = getproperty(x, Symbol(k)) + PyJuliaRawValue_New(v) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlraw_setattro(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericSetAttr(xo, ko, vo) + if ism1(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to set the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return Cint(-1) + ism1(PyObject_Convert(vo, Any)) && return Cint(-1) + v = takeresult(Any) + try + setproperty!(x, Symbol(k), v) + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +pyjlraw_dir(xo::PyPtr, _::PyPtr) = begin + fo = PyObject_GetAttrString(PyJuliaBaseValue_Type(), "__dir__") + isnull(fo) && return PyPtr() + ro = PyObject_CallNice(fo, PyObjectRef(xo)) + Py_DecRef(fo) + isnull(ro) && return PyPtr() + x = PyJuliaValue_GetValue(xo) + ks = try + collect(map(string, propertynames(x)))::Vector{String} + catch err + Py_DecRef(ro) + PyErr_SetJuliaError(err) + return PyPtr() + end + for k in ks + ko = PyUnicode_From(k) + isnull(ko) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, ko) + Py_DecRef(ko) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + return ro +end + +pyjlraw_call(fo::PyPtr, argso::PyPtr, kwargso::PyPtr) = begin + f = PyJuliaValue_GetValue(fo) + if isnull(argso) + args = Vector{Any}() + else + ism1(PyObject_Convert(argso, Vector{Any})) && return PyPtr() + args = takeresult(Vector{Any}) + end + if isnull(kwargso) + kwargs = Dict{Symbol, Any}() + else + ism1(PyObject_Convert(kwargso, Dict{Symbol, Any})) && return PyPtr() + kwargs = takeresult(Dict{Symbol, Any}) + end + try + x = f(args...; kwargs...) + PyJuliaRawValue_New(x) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlraw_length(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + Py_ssize_t(length(x)) +catch err + PyErr_SetJuliaError(err) + Py_ssize_t(-1) +end + +pyjlraw_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + if PyTuple_Check(ko) + ism1(PyObject_Convert(ko, Tuple)) && return PyPtr() + k = takeresult(Tuple) + try + PyJuliaRawValue_New(x[k...]) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + else + ism1(PyObject_Convert(ko, Any)) && return PyPtr() + k = takeresult(Any) + try + PyJuliaRawValue_New(x[k]) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + end +end + +pyjlraw_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + ism1(PyObject_Convert(vo, Any)) && return PyPtr() + v = takeresult(Any) + if PyTuple_Check(ko) + ism1(PyObject_Convert(ko, Tuple)) && return PyPtr() + k = takeresult(Tuple) + try + x[k...] = v + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end + else + ism1(PyObject_Convert(ko, Any)) && return PyPtr() + k = takeresult(Any) + try + x[k] = v + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end + end +end + +PyJuliaRawValue_New(x) = begin + t = PyJuliaRawValue_Type() + isnull(t) && return PyPtr() + PyJuliaValue_New(t, x) +end diff --git a/src/cpython/list.jl b/src/cpython/list.jl index 56409c81..7e1fff71 100644 --- a/src/cpython/list.jl +++ b/src/cpython/list.jl @@ -1,3 +1,31 @@ @cdef :PyList_New PyPtr (Py_ssize_t,) @cdef :PyList_Append Cint (PyPtr, PyPtr) @cdef :PyList_AsTuple PyPtr (PyPtr,) + +const PyList_Type__ref = Ref(PyPtr()) +PyList_Type() = pyglobal(PyList_Type__ref, :PyList_Type) + +PyList_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LIST_SUBCLASS) + +PyList_CheckExact(o) = Py_TypeCheckExact(o, PyList_Type()) + +PyList_From(xs::Union{Tuple,AbstractVector}) = PyList_FromIter(xs) + +PyList_FromIter(xs) = begin + r = PyList_New(0) + isnull(r) && return PyPtr() + try + for (i,x) in enumerate(xs) + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(r); return PyPtr()) + err = PyList_Append(r, xo) + Py_DecRef(xo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end diff --git a/src/cpython/newtype.jl b/src/cpython/newtype.jl new file mode 100644 index 00000000..f6e9fe08 --- /dev/null +++ b/src/cpython/newtype.jl @@ -0,0 +1,125 @@ +### CACHE + +cacheptr!(c, x::Ptr) = x +cacheptr!(c, x::String) = (push!(c, x); pointer(x)) +cacheptr!(c, x::AbstractString) = cacheptr!(c, String(x)) +cacheptr!(c, x::Array) = (push!(c, x); pointer(x)) +cacheptr!(c, x::AbstractArray) = cacheptr!(c, Array(x)) +cacheptr!(c, x::PyObject) = (push!(c, x); pyptr(x)) +cacheptr!(c, x::Base.CFunction) = (push!(c, x); Base.unsafe_convert(Ptr{Cvoid}, x)) + +cachestrptr!(c, x::Ptr) = x +cachestrptr!(c, x) = cacheptr!(c, string(x)) + +macro cachefuncptr!(c, f, R, Ts) + quote + let c=$(esc(c)), f=$(esc(f)) + if f isa Ptr + f + elseif f isa Base.CFunction + cacheptr!(c, f) + else + cacheptr!(c, @cfunction($(Expr(:$, :f)), $(esc(R)), ($(map(esc, Ts.args)...),))) + end + end + end +end + +### PROTOCOLS + +PyNumberMethods_Create(c, x::PyNumberMethods) = x +PyNumberMethods_Create(c; opts...) = C.PyNumberMethods(; [k => (v isa Ptr ? v : v isa Base.CFunction ? cacheptr!(c, v) : error()) for (k,v) in pairs(opts)]...) +PyNumberMethods_Create(c, x::Dict) = PyNumberMethods_Create(c; x...) +PyNumberMethods_Create(c, x::NamedTuple) = PyNumberMethods_Create(c; x...) + +PyMappingMethods_Create(c, x::PyMappingMethods) = x +PyMappingMethods_Create(c; length=C_NULL, subscript=C_NULL, ass_subscript=C_NULL) = + PyMappingMethods( + length = @cachefuncptr!(c, length, Py_ssize_t, (PyPtr,)), + subscript = @cachefuncptr!(c, subscript, PyPtr, (PyPtr, PyPtr)), + ass_subscript = @cachefuncptr!(c, ass_subscript, Cint, (PyPtr, PyPtr, PyPtr)), + ) +PyMappingMethods_Create(c, x::Dict) = PyMappingMethods_Create(c; x...) +PyMappingMethods_Create(c, x::NamedTuple) = PyMappingMethods_Create(c; x...) + +PySequenceMethods_Create(c, x::PySequenceMethods) = x +PySequenceMethods_Create(c; opts...) = C.PySequenceMethods(; [k => (v isa Ptr ? v : v isa Base.CFunction ? cacheptr!(c, v) : error()) for (k,v) in pairs(opts)]...) +PySequenceMethods_Create(c, x::Dict) = PySequenceMethods_Create(c; x...) +PySequenceMethods_Create(c, x::NamedTuple) = PySequenceMethods_Create(c; x...) + +PyBufferProcs_Create(c, x::PyBufferProcs) = x +PyBufferProcs_Create(c; get=C_NULL, release=C_NULL) = + PyBufferProcs( + get = @cachefuncptr!(c, get, Cint, (PyPtr, Ptr{Py_buffer}, Cint)), + release = @cachefuncptr!(c, release, Cvoid, (PyPtr, Ptr{Py_buffer})), + ) +PyBufferProcs_Create(c, x::Dict) = PyBufferProcs_Create(c; x...) +PyBufferProcs_Create(c, x::NamedTuple) = PyBufferProcs_Create(c; x...) + +PyMethodDef_Create(c, x::PyMethodDef) = x +PyMethodDef_Create(c; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = + PyMethodDef( + name = cachestrptr!(c, name), + meth = iszero(flags & Py_METH_VARARGS) ? @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr)) : @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr, PyPtr)), + flags = flags, + doc = cachestrptr!(c, doc), + ) +PyMethodDef_Create(c, x::Dict) = PyMethodDef_Create(c; x...) +PyMethodDef_Create(c, x::NamedTuple) = PyMethodDef_Create(c; x...) + +PyGetSetDef_Create(c, x::PyGetSetDef) = x +PyGetSetDef_Create(c; name=C_NULL, get=C_NULL, set=C_NULL, doc=C_NULL, closure=C_NULL) = + PyGetSetDef( + name = cachestrptr!(c, name), + get = @cachefuncptr!(c, get, PyPtr, (PyPtr, Ptr{Cvoid})), + set = @cachefuncptr!(c, set, Cint, (PyPtr, PyPtr, Ptr{Cvoid})), + doc = cachestrptr!(c, doc), + closure = closure, + ) +PyGetSetDef_Create(c, x::Dict) = PyGetSetDef_Create(c; x...) +PyGetSetDef_Create(c, x::NamedTuple) = PyGetSetDef_Create(c; x...) + +PyType_Create(c; + type=C_NULL, name, as_number=C_NULL, as_mapping=C_NULL, as_sequence=C_NULL, + as_buffer=C_NULL, methods=C_NULL, getset=C_NULL, dealloc=C_NULL, getattr=C_NULL, + setattr=C_NULL, repr=C_NULL, hash=C_NULL, call=C_NULL, str=C_NULL, getattro=C_NULL, + setattro=C_NULL, doc=C_NULL, iter=C_NULL, iternext=C_NULL, opts... +) = begin + type = cacheptr!(c, type) + name = cachestrptr!(c, name) + as_number = as_number isa Ptr ? as_number : cacheptr!(c, fill(PyNumberMethods_Create(c, as_number))) + as_mapping = as_mapping isa Ptr ? as_mapping : cacheptr!(c, fill(PyMappingMethods_Create(c, as_mapping))) + as_sequence = as_sequence isa Ptr ? as_sequence : cacheptr!(c, fill(PySequenceMethods_Create(c, as_sequence))) + as_buffer = as_buffer isa Ptr ? as_buffer : cacheptr!(c, fill(PyBufferProcs_Create(c, as_buffer))) + methods = + if methods isa Ptr + methods + else + cacheptr!(c, [[PyMethodDef_Create(c, m) for m in methods]; PyMethodDef()]) + end + getset = + if getset isa Ptr + getset + else + cacheptr!(c, [[PyGetSetDef_Create(c, m) for m in getset]; PyGetSetDef()]) + end + dealloc = @cachefuncptr!(c, dealloc, Cvoid, (PyPtr,)) + getattr = @cachefuncptr!(c, getattr, PyPtr, (PyPtr, Cstring)) + setattr = @cachefuncptr!(c, setattr, Cint, (PyPtr, Cstring, PyPtr)) + repr = @cachefuncptr!(c, repr, PyPtr, (PyPtr,)) + hash = @cachefuncptr!(c, hash, Py_hash_t, (PyPtr,)) + call = @cachefuncptr!(c, call, PyPtr, (PyPtr, PyPtr, PyPtr)) + str = @cachefuncptr!(c, str, PyPtr, (PyPtr,)) + getattro = @cachefuncptr!(c, getattro, PyPtr, (PyPtr, PyPtr)) + setattro = @cachefuncptr!(c, setattro, Cint, (PyPtr, PyPtr, PyPtr)) + doc = cachestrptr!(c, doc) + iter = @cachefuncptr!(c, iter, PyPtr, (PyPtr,)) + iternext = @cachefuncptr!(c, iter, PyPtr, (PyPtr,)) + PyTypeObject(; + ob_base=PyVarObject(ob_base=PyObject(type=type)), name=name, as_number=as_number, + as_mapping=as_mapping, as_sequence=as_sequence, as_buffer=as_buffer, + methods=methods, getset=getset, dealloc=dealloc, getattr=getattr, setattr=setattr, + repr=repr, hash=hash, call=call, str=str, getattro=getattro, setattro=setattro, + iter=iter, iternext=iternext, opts... + ) +end diff --git a/src/cpython/object.jl b/src/cpython/object.jl index d35161a9..4d3679fb 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -29,14 +29,23 @@ @cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) @cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) +const PyObject_Type__ref = Ref(PyPtr()) +PyObject_Type() = pyglobal(PyObject_Type__ref, :PyBaseObject_Type) + PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) PyObject_From(x::Nothing) = PyNone_New() +PyObject_From(x::Missing) = PyNone_New() PyObject_From(x::Bool) = PyBool_From(x) PyObject_From(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = PyLong_From(x) +PyObject_From(x::Rational{<:Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyFraction_From(x) PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) PyObject_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_From(x) PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) +PyObject_From(x::Char) = PyUnicode_From(string(x)) PyObject_From(x::Tuple) = PyTuple_From(x) +PyObject_From(x::AbstractVector) = PyList_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED +PyObject_From(x::AbstractDict) = PyDict_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED +PyObject_From(x::AbstractSet) = PySet_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED PyObject_From(x::AbstractRange{<:Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyRange_From(x) PyObject_From(x::T) where {T} = if ispyreftype(T) diff --git a/src/cpython/set.jl b/src/cpython/set.jl index d7083a51..aff24111 100644 --- a/src/cpython/set.jl +++ b/src/cpython/set.jl @@ -1,2 +1,54 @@ @cdef :PySet_New PyPtr (PyPtr,) @cdef :PyFrozenSet_New PyPtr (PyPtr,) +@cdef :PySet_Add Cint (PyPtr, PyPtr) + +const PySet_Type__ref = Ref(PyPtr()) +PySet_Type() = pyglobal(PySet_Type__ref, :PySet_Type) + +PySet_Check(o) = Py_TypeCheckFast(o, PySet_Type()) + +PySet_CheckExact(o) = Py_TypeCheckExact(o, PySet_Type()) + +const PyFrozenSet_Type__ref = Ref(PyPtr()) +PyFrozenSet_Type() = pyglobal(PyFrozenSet_Type__ref, :PyFrozenSet_Type) + +PyFrozenSet_Check(o) = Py_TypeCheckFast(o, PyFrozenSet_Type()) + +PyFrozenSet_CheckExact(o) = Py_TypeCheckExact(o, PyFrozenSet_Type()) + +PyAnySet_Check(o) = PySet_Check(o) || PyFrozenSet_Check(o) + +PyAnySet_CheckExact(o) = PySet_CheckExact(o) || PyFrozenSet_CheckExact(o) + +_PySet_FromIter(r::PyPtr, xs) = begin + try + for x in xs + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(r); return PyPtr()) + err = PySet_Add(r, xo) + Py_DecRef(xo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end + +PySet_FromIter(xs) = begin + r = PySet_New(C_NULL) + isnull(r) && return PyPtr() + _PySet_FromIter(r, xs) +end + +PyFrozenSet_FromIter(xs) = begin + r = PyFrozenSet_New(C_NULL) + isnull(r) && return PyPtr() + _PySet_FromIter(r, xs) +end + +PySet_From(x::Union{AbstractSet,AbstractVector,Tuple}) = PySet_FromIter(x) + +PyFrozenSet_From(x::Union{AbstractSet,AbstractVector,Tuple}) = PyFrozenSet_FromIter(x) diff --git a/src/cpython/slice.jl b/src/cpython/slice.jl new file mode 100644 index 00000000..e71ceb9f --- /dev/null +++ b/src/cpython/slice.jl @@ -0,0 +1,17 @@ +@cdef :PySlice_New PyPtr (PyPtr, PyPtr, PyPtr) + +const PySlice_Type__ref = Ref(PyPtr()) +PySlice_Type() = pyglobal(PySlice_Type__ref, :PySlice_Type) + +PySlice_Check(o) = Py_TypeCheck(o, PySlice_Type()) + +PySlice_CheckExact(o) = Py_TypeCheckExact(o, PySlice_Type()) + +### ELLIPSIS + +const Py_Ellipsis__ref = Ref(PyPtr()) +Py_Ellipsis() = pyglobal(Py_Ellipsis__ref, :_Py_EllipsisObject) + +PyEllipsis_Check(o) = Py_Is(o, Py_Ellipsis()) + +PyEllipsis_New() = (o=Py_Ellipsis(); Py_IncRef(o); o) diff --git a/src/cpython/str.jl b/src/cpython/str.jl index 2ead90f7..26a77982 100644 --- a/src/cpython/str.jl +++ b/src/cpython/str.jl @@ -11,34 +11,43 @@ PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) -PyUnicode_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin +PyUnicode_AsString(o) = begin b = PyUnicode_AsUTF8String(o) - isnull(b) && return -1 - r = PyBytes_TryConvertRule_string(b, String, String) + isnull(b) && return "" + r = PyBytes_AsString(b) Py_DecRef(b) - r == 1 || return r - moveresult(String, T) + r end -PyUnicode_TryConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = begin +PyUnicode_AsVector(o, ::Type{T}=UInt8) where {T} = begin b = PyUnicode_AsUTF8String(o) - isnull(b) && return -1 - r = PyBytes_TryConvertRule_vector(b, S, S) + isnull(b) && return T[] + r = PyBytes_AsVector(b, T) Py_DecRef(b) - r == 1 || return r - moveresult(S, T) + r +end + +PyUnicode_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + putresult(T, r) +end + +PyUnicode_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T, X} = begin + r = PyUnicode_AsVector(o, X) + isempty(r) && PyErr_IsSet() && return -1 + putresult(T, r) end PyUnicode_TryConvertRule_symbol(o, ::Type{T}, ::Type{Symbol}) where {T} = begin - r = PyUnicode_TryConvertRule_string(o, String, String) - r == 1 || return r - putresult(T, Symbol(takeresult(String))) + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + putresult(T, Symbol(r)) end PyUnicode_TryConvertRule_char(o, ::Type{T}, ::Type{Char}) where {T} = begin - r = PyUnicode_TryConvertRule_string(o, String, String) - r == 1 || return r - s = takeresult(String) - length(s) == 1 || return 0 - putresult(T, first(s)) + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + length(r) == 1 || return 0 + putresult(T, first(r)) end diff --git a/src/cpython/tuple.jl b/src/cpython/tuple.jl index b515b579..9af52363 100644 --- a/src/cpython/tuple.jl +++ b/src/cpython/tuple.jl @@ -3,7 +3,16 @@ @cdef :PyTuple_GetItem PyPtr (PyPtr, Py_ssize_t) @cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) -function PyTuple_From(xs::Tuple) +const PyTuple_Type__ref = Ref(PyPtr()) +PyTuple_Type() = pyglobal(PyTuple_Type__ref, :PyTuple_Type) + +PyTuple_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_TUPLE_SUBCLASS) + +PyTuple_CheckExact(o) = Py_TypeCheckExact(o, PyTuple_Type()) + +PyTuple_From(x::Union{Tuple,AbstractVector}) = PyTuple_FromIter(x) + +PyTuple_FromIter(xs::Tuple) = begin t = PyTuple_New(length(xs)) isnull(t) && return PyPtr() for (i,x) in enumerate(xs) @@ -14,3 +23,11 @@ function PyTuple_From(xs::Tuple) end return t end + +PyTuple_FromIter(xs) = begin + y = PyList_FromIter(xs) + isnull(y) && return PyPtr() + t = PyList_AsTuple(y) + Py_DecRef(y) + return t +end diff --git a/src/init.jl b/src/init.jl index 8a4a16f2..4ac73576 100644 --- a/src/init.jl +++ b/src/init.jl @@ -114,86 +114,99 @@ function __init__() # Start the interpreter and register exit hooks C.Py_InitializeEx(0) CONFIG.isinitialized = true - # check(C.Py_AtExit(@cfunction(()->(CONFIG.isinitialized = false; nothing), Cvoid, ()))) - # atexit() do - # CONFIG.isinitialized = false - # check(C.Py_FinalizeEx()) - # end + check(C.Py_AtExit(@cfunction(()->(CONFIG.isinitialized = false; nothing), Cvoid, ()))) + atexit() do + CONFIG.isinitialized = false + checkm1(C.Py_FinalizeEx()) + end end end - C.PyObject_TryConvert_AddRule("builtins.object", PyObject, CTryConvertRule_wrapref, -100) - C.PyObject_TryConvert_AddRule("builtins.object", PyRef, CTryConvertRule_wrapref, -100) - C.PyObject_TryConvert_AddRule("collections.abc.Sequence", PyList, CTryConvertRule_wrapref, 100) - C.PyObject_TryConvert_AddRule("collections.abc.Set", PySet, CTryConvertRule_wrapref, 100) - C.PyObject_TryConvert_AddRule("collections.abc.Mapping", PyDict, CTryConvertRule_wrapref, 100) - - # with_gil() do - - # if !CONFIG.isembedded - # # Some modules expect sys.argv to be set - # pysysmodule.argv = pylist([""; ARGS]) - - # # Some modules test for interactivity by checking if sys.ps1 exists - # if isinteractive() && !pyhasattr(pysysmodule, "ps1") - # pysysmodule.ps1 = ">>> " - # end - # end - - # # Is this the same Python as in Conda? - # if !CONFIG.isconda && - # haskey(ENV, "CONDA_PREFIX") && isdir(ENV["CONDA_PREFIX"]) && - # haskey(ENV, "CONDA_PYTHON_EXE") && isfile(ENV["CONDA_PYTHON_EXE"]) && - # realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? pyconvert(String, pysysmodule.executable) : CONFIG.exepath) - - # CONFIG.isconda = true - # CONFIG.condaenv = ENV["CONDA_PREFIX"] - # CONFIG.exepath === nothing && (CONFIG.exepath = pyconvert(String, pysysmodule.executable)) - # end - - # # Get the python version - # CONFIG.version = let (a,b,c,d,e) = pyconvert(Tuple{Int,Int,Int,String,Int}, pysysmodule.version_info) - # VersionNumber(a, b, c, (d,), (e,)) - # end - # v"2" < CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version.major).$(CONFIG.version.minor) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") - - # # EXPERIMENTAL: hooks to perform actions when certain modules are loaded - # if !CONFIG.isembedded - # py""" - # import sys - # class JuliaCompatHooks: - # def __init__(self): - # self.hooks = {} - # def find_module(self, name, path=None): - # hs = self.hooks.get(name) - # if hs is not None: - # for h in hs: - # h() - # def add_hook(self, name, h): - # if name not in self.hooks: - # self.hooks[name] = [h] - # else: - # self.hooks[name].append(h) - # if name in sys.modules: - # h() - # JULIA_COMPAT_HOOKS = JuliaCompatHooks() - # sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) - - # # Before Qt is loaded, fix the path used to look up its plugins - # qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) - # JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) - # """ - - # @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin - # IJulia.push_postexecute_hook() do - # if CONFIG.pyplotautoshow && "matplotlib.pyplot" in pysysmodule.modules - # pyplotshow() - # end - # end - # end - # end - # end + C.PyObject_TryConvert_AddRules("builtins.object", [ + (PyObject, CTryConvertRule_wrapref, -100), + (PyRef, CTryConvertRule_wrapref, -200), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Sequence", [ + (PyList, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Set", [ + (PySet, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Mapping", [ + (PyDict, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyBuffer, CTryConvertRule_wrapref, -200), + ]) + + with_gil() do + + @pyg `import sys, os` + + if !CONFIG.isembedded + @py ``` + # Some modules expect sys.argv to be set + # TODO: Append ARGS + sys.argv = [""] + + # Some modules test for interactivity by checking if sys.ps1 exists + if $(isinteractive()) and not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + ``` + end + + # Is this the same Python as in Conda? + if !CONFIG.isconda && + haskey(ENV, "CONDA_PREFIX") && isdir(ENV["CONDA_PREFIX"]) && + haskey(ENV, "CONDA_PYTHON_EXE") && isfile(ENV["CONDA_PYTHON_EXE"]) && + realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? @pyv(`sys.executable`::String) : CONFIG.exepath) + + CONFIG.isconda = true + CONFIG.condaenv = ENV["CONDA_PREFIX"] + CONFIG.exepath === nothing && (CONFIG.exepath = @pyv(`sys.executable`::String)) + end + + # Get the python version + CONFIG.version = let (a,b,c,d,e) = @pyv(`sys.version_info`::Tuple{Int,Int,Int,String,Int}) + VersionNumber(a, b, c, (d,), (e,)) + end + v"3" ≤ CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") + + # EXPERIMENTAL: hooks to perform actions when certain modules are loaded + if !CONFIG.isembedded + # py""" + # import sys + # class JuliaCompatHooks: + # def __init__(self): + # self.hooks = {} + # def find_module(self, name, path=None): + # hs = self.hooks.get(name) + # if hs is not None: + # for h in hs: + # h() + # def add_hook(self, name, h): + # if name not in self.hooks: + # self.hooks[name] = [h] + # else: + # self.hooks[name].append(h) + # if name in sys.modules: + # h() + # JULIA_COMPAT_HOOKS = JuliaCompatHooks() + # sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) + + # # Before Qt is loaded, fix the path used to look up its plugins + # qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) + # JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) + # JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) + # """ + + @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin + IJulia.push_postexecute_hook() do + CONFIG.pyplotautoshow && pyplotshow() + end + end + end + end end diff --git a/src/julia.jl b/src/julia.jl new file mode 100644 index 00000000..1fc4287e --- /dev/null +++ b/src/julia.jl @@ -0,0 +1,28 @@ +pyjlbasetype(::Type{T}) where {T} = checknullconvert(T, C.PyJuliaBaseValue_Type()) +pyjlbasetype() = pyjlbasetype(PyObject) + +pyjlrawtype(::Type{T}) where {T} = checknullconvert(T, C.PyJuliaRawValue_Type()) +pyjlrawtype() = pyjlrawtype(PyObject) + +""" + pyjlraw([T=PyObject,] x) + +Wrap `x` as a Python `julia.RawValue` object. +""" +pyjlraw(::Type{T}, x) where {T} = checknullconvert(T, C.PyJuliaRawValue_New(x)) +pyjlraw(x) = pyjlraw(PyObject, x) +export pyjlraw + +""" + pyjlgetvalue() +""" +pyjlgetvalue(o) = pyisjl(o) ? cpyop(C.PyJuliaValue_GetValue, o) : error("Not a Julia value") +export pyjlgetvalue + +""" + pyisjl(o) + +True if `o` is a `julia.ValueBase` object. +""" +pyisjl(o) = cpyop(C.PyJuliaValue_Check, o) +export pyisjl diff --git a/src/matplotlib.jl b/src/matplotlib.jl new file mode 100644 index 00000000..31d184ec --- /dev/null +++ b/src/matplotlib.jl @@ -0,0 +1,53 @@ +""" + pyplotshow([fig]; close=true, [format]) + +Show the matplotlib/pyplot/seaborn/etc figure `fig`, or all open figures if not given. + +If `close` is true, the figure is also closed. + +The `format` specifies the file format of the generated image. By default this is `pyplot.rcParams["savefig.format"]`. +""" +function pyplotshow(fig; close::Bool=true, format::String="") + @py ``` + import matplotlib.pyplot as plt, io + fig = $fig + if not isinstance(fig, plt.Figure): + fig = plt.figure(fig) + buf = io.BytesIO() + format = $format + if not format: + format = plt.rcParams.get("savefig.format", "png") + if format not in ["png", "jpg", "jpeg", "tif", "tiff", "svg", "pdf"]: + raise ValueError("Unsupported format: {}".format(format)) + fig.savefig(buf, format=format) + $(data::Vector{UInt8}) = buf.getvalue() + if $close: + plt.close(fig) + $(format::String) = format + ``` + if format == "png" + display(MIME"image/png"(), data) + elseif format in ("jpg", "jpeg") + display(MIME"image/jpeg"(), data) + elseif format in ("tif", "tiff") + display(MIME"image/tiff"(), data) + elseif format == "svg" + display(MIME"image/svg+xml"(), String(data)) + elseif format == "pdf" + display(MIME"application/pdf"(), data) + else + error() + end + nothing +end +function pyplotshow(; opts...) + @py ``` + import sys + plt = sys.modules.get("matplotlib.pyplot", None) + $(fignums::Vector{Int}) = [] if plt is None else plt.get_fignums() + ``` + for fig in fignums + pyplotshow(fig; opts...) + end +end +export pyplotshow diff --git a/src/old/matplotlib.jl b/src/old/matplotlib.jl deleted file mode 100644 index 80008a80..00000000 --- a/src/old/matplotlib.jl +++ /dev/null @@ -1,39 +0,0 @@ -pymatplotlib = pylazyobject(() -> pyimport("matplotlib")) -pyplot = pylazyobject(() -> pyimport("matplotlib.pyplot")) - -""" - pyplotshow([fig]; close=true, [format]) - -Show the matplotlib/pyplot/seaborn/etc figure `fig`, or all open figures if not given. - -If `close` is true, the figure is also closed. - -The `format` specifies the file format of the generated image. By default this is `pyplot.rcParams["savefig.format"]`. -""" -function pyplotshow(fig; close::Bool=true, format::String=pyplot.rcParams.get("savefig.format", "png").jl!s) - fig = pyisinstance(fig, pyplot.Figure) ? PyObject(fig) : pyplot.figure(fig) - io = IOBuffer() - fig.savefig(io, format=format) - data = take!(io) - if format == "png" - display(MIME"image/png"(), data) - elseif format in ("jpg", "jpeg") - display(MIME"image/jpeg"(), data) - elseif format in ("tif", "tiff") - display(MIME"image/tiff"(), data) - elseif format == "svg" - display(MIME"image/svg+xml"(), String(data)) - elseif format == "pdf" - display(MIME"application/pdf"(), data) - else - error("Unsupported format: $(repr(format)) (try one of: png, jpg, jpeg, tif, tiff, svg, xml)") - end - close && pyplot.close(fig) - nothing -end -function pyplotshow(; opts...) - for fig in pyplot.get_fignums() - pyplotshow(fig; opts...) - end -end -export pyplotshow diff --git a/src/old/newtype.jl b/src/old/newtype.jl deleted file mode 100644 index cee95f32..00000000 --- a/src/old/newtype.jl +++ /dev/null @@ -1,79 +0,0 @@ -### CACHE - -cacheptr!(c, x::Ptr) = x -cacheptr!(c, x::String) = (push!(c, x); pointer(x)) -cacheptr!(c, x::AbstractString) = cacheptr!(c, String(x)) -cacheptr!(c, x::Array) = (push!(c, x); pointer(x)) -cacheptr!(c, x::AbstractArray) = cacheptr!(c, Array(x)) -cacheptr!(c, x::PyObject) = (push!(c, x); pyptr(x)) -cacheptr!(c, x::Base.CFunction) = (push!(c, x); Base.unsafe_convert(Ptr{Cvoid}, x)) - -### PROTOCOLS - -cpynewnumbermethods!(c, x::C.PyNumberMethods) = x -cpynewnumbermethods!(c; opts...) = C.PyNumberMethods(; opts...) -cpynewnumbermethods!(c, x::Dict) = cpynewnumbermethods!(c; x...) -cpynewnumbermethods!(c, x::NamedTuple) = cpynewnumbermethods!(c; x...) - -cpynewmappingmethods!(c, x::C.PyMappingMethods) = x -cpynewmappingmethods!(c; opts...) = C.PyMappingMethods(; opts...) -cpynewmappingmethods!(c, x::Dict) = cpynewmappingmethods!(c; x...) -cpynewmappingmethods!(c, x::NamedTuple) = cpynewmappingmethods!(c; x...) - -cpynewsequencemethods!(c, x::C.PySequenceMethods) = x -cpynewsequencemethods!(c; opts...) = C.PySequenceMethods(; opts...) -cpynewsequencemethods!(c, x::Dict) = cpynewsequencemethods!(c; x...) -cpynewsequencemethods!(c, x::NamedTuple) = cpynewsequencemethods!(c; x...) - -cpynewbufferprocs!(c, x::C.PyBufferProcs) = x -cpynewbufferprocs!(c; opts...) = C.PyBufferProcs(; opts...) -cpynewbufferprocs!(c, x::Dict) = cpynewbufferprocs!(c; x...) -cpynewbufferprocs!(c, x::NamedTuple) = cpynewbufferprocs!(c; x...) - -cpynewmethod!(c, x::C.PyMethodDef) = x -cpynewmethod!(c; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = - C.PyMethodDef( - name = name isa Ptr ? name : cacheptr!(c, string(name)), - meth = meth, - flags = flags, - doc = doc isa Ptr ? doc : cacheptr!(c, string(doc)), - ) -cpynewmethod!(c, x::Dict) = cpynewmethod!(c; x...) -cpynewmethod!(c, x::NamedTuple) = cpynewmethod!(c; x...) - -cpynewgetset!(c, x::C.PyGetSetDef) = x -cpynewgetset!(c; name=C_NULL, get=C_NULL, set=C_NULL, doc=C_NULL, closure=C_NULL) = - C.PyGetSetDef( - name = name isa Ptr ? name : cacheptr!(c, string(name)), - get = get, - set = set, - doc = doc isa Ptr ? doc : cacheptr!(c, string(doc)), - closure = closure, - ) -cpynewgetset!(c, x::Dict) = cpynewgetset!(c; x...) -cpynewgetset!(c, x::NamedTuple) = cpynewgetset!(c; x...) - -### NEW TYPE - -function cpynewtype!(c; type=C_NULL, name, as_number=C_NULL, as_mapping=C_NULL, as_sequence=C_NULL, as_buffer=C_NULL, methods=C_NULL, getset=C_NULL, opts...) - type = cacheptr!(c, type) - name = cacheptr!(c, name) - as_number = as_number isa Ptr ? as_number : cacheptr!(c, fill(cpynewnumbermethods!(c, as_number))) - as_mapping = as_mapping isa Ptr ? as_mapping : cacheptr!(c, fill(cpynewmappingmethods!(c, as_mapping))) - as_sequence = as_sequence isa Ptr ? as_sequence : cacheptr!(c, fill(cpynewsequencemethods!(c, as_sequence))) - as_buffer = as_buffer isa Ptr ? as_buffer : cacheptr!(c, fill(cpynewbufferprocs!(c, as_buffer))) - methods = - if methods isa Ptr - methods - else - cacheptr!(c, [[cpynewmethod!(c, m) for m in methods]; C.PyMethodDef()]) - end - getset = - if getset isa Ptr - getset - else - cacheptr!(c, [[cpynewgetset!(c, m) for m in getset]; C.PyGetSetDef()]) - end - newopts = Dict(k => (v isa Union{AbstractString,AbstractArray,PyObject,Base.CFunction} ? cacheptr!(c, v) : v) for (k,v) in pairs(opts)) - C.PyTypeObject(; ob_base=C.PyVarObject(ob_base=C.PyObject(type=type)), name=name, as_number=as_number, as_mapping=as_mapping, as_sequence=as_sequence, as_buffer=as_buffer, methods, getset, newopts...) -end From 5d10a34b2cfce3b8a47a877c926703ec3d67a683 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Tue, 22 Dec 2020 18:35:01 +0000 Subject: [PATCH 11/14] dev --- src/PyIO.jl | 2 +- src/PyIterable.jl | 2 +- src/PyList.jl | 2 +- src/PyObject.jl | 5 ++- src/PyPandasDataFrame.jl | 2 +- src/PyRef.jl | 2 +- src/PySet.jl | 2 +- src/cpython/CPython.jl | 10 ++++- src/cpython/arg.jl | 8 ++-- src/cpython/bool.jl | 6 +-- src/cpython/bytes.jl | 8 ++-- src/cpython/collections.jl | 64 ++++++++++++++++++-------------- src/cpython/complex.jl | 8 ++-- src/cpython/ctypes.jl | 4 +- src/cpython/float.jl | 8 ++-- src/cpython/int.jl | 18 ++++----- src/cpython/juliabase.jl | 12 ++++-- src/cpython/juliaerror.jl | 34 ++++++++++++++++- src/cpython/juliaraw.jl | 24 ++++++------ src/cpython/newtype.jl | 24 +++++++++--- src/cpython/none.jl | 11 +++++- src/cpython/numpy.jl | 4 +- src/cpython/object.jl | 76 +++++++++++++++++++------------------- src/cpython/range.jl | 8 ++-- src/cpython/str.jl | 16 ++++---- src/cpython/type.jl | 12 +++--- src/julia.jl | 9 +++++ src/utils.jl | 19 +++------- 28 files changed, 238 insertions(+), 162 deletions(-) diff --git a/src/PyIO.jl b/src/PyIO.jl index 8e9ea163..b1c719f0 100644 --- a/src/PyIO.jl +++ b/src/PyIO.jl @@ -37,7 +37,7 @@ export PyIO ispyreftype(::Type{PyIO}) = true pyptr(io::PyIO) = pyptr(io.ref) Base.unsafe_convert(::Type{CPyPtr}, io::PyIO) = checknull(pyptr(io)) -C.PyObject_TryConvert__initial(o, ::Type{PyIO}) = C.putresult(PyIO, PyIO(pyborrowedref(o))) +C.PyObject_TryConvert__initial(o, ::Type{PyIO}) = C.putresult(PyIO(pyborrowedref(o))) """ PyIO(f, o; ...) diff --git a/src/PyIterable.jl b/src/PyIterable.jl index 08f6105e..00f863b8 100644 --- a/src/PyIterable.jl +++ b/src/PyIterable.jl @@ -13,7 +13,7 @@ export PyIterable ispyreftype(::Type{<:PyIterable}) = true pyptr(x::PyIterable) = pyptr(x.ref) Base.unsafe_convert(::Type{CPyPtr}, x::PyIterable) = checknull(pyptr(x)) -C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyIterable} = C.putresult(T, T(pyborrowedref(o))) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyIterable} = C.putresult(T(pyborrowedref(o))) Base.length(x::PyIterable) = Int(pylen(x)) diff --git a/src/PyList.jl b/src/PyList.jl index 8d744292..ca890bc5 100644 --- a/src/PyList.jl +++ b/src/PyList.jl @@ -23,7 +23,7 @@ pyptr(x::PyList) = begin ptr end Base.unsafe_convert(::Type{CPyPtr}, x::PyList) = checknull(pyptr(x)) -C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyList} = C.putresult(T, T(pyborrowedref(o))) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyList} = C.putresult(T(pyborrowedref(o))) # Base.length(x::PyList) = @pyv `len($x)`::Int Base.length(x::PyList) = Int(pylen(x)) diff --git a/src/PyObject.jl b/src/PyObject.jl index 02731fee..f5dc3a4b 100644 --- a/src/PyObject.jl +++ b/src/PyObject.jl @@ -27,11 +27,14 @@ pynewobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyOb pyborrowedobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyObject(Val(:nocopy), pyborrowedref(p)) pylazyobject(mk) = PyObject(Val(:lazy), mk) -C.PyObject_TryConvert__initial(o, ::Type{PyObject}) = C.putresult(PyObject, pyborrowedobject(o)) +C.PyObject_TryConvert__initial(o, ::Type{PyObject}) = C.putresult(pyborrowedobject(o)) +Base.convert(::Type{Any}, x::PyObject) = x Base.convert(::Type{PyObject}, x::PyObject) = x Base.convert(::Type{PyObject}, x) = PyObject(x) +Base.convert(::Type{T}, x::PyObject) where {T} = pyconvert(T, x) + ### Cache some common values const _pynone = pylazyobject(() -> pynone(PyRef)) diff --git a/src/PyPandasDataFrame.jl b/src/PyPandasDataFrame.jl index 3ce5306a..b3ab4db5 100644 --- a/src/PyPandasDataFrame.jl +++ b/src/PyPandasDataFrame.jl @@ -74,7 +74,7 @@ export PyPandasDataFrame ispyreftype(::Type{PyPandasDataFrame}) = true pyptr(df::PyPandasDataFrame) = df.ref Base.unsafe_convert(::Type{CPyPtr}, df::PyPandasDataFrame) = checknull(pyptr(df)) -C.PyObject_TryConvert__initial(o, ::Type{PyPandasDataFrame}) = C.putresult(PyPandasDataFrame, PyPandasDataFrame(pyborrowedref(o))) +C.PyObject_TryConvert__initial(o, ::Type{PyPandasDataFrame}) = C.putresult(PyPandasDataFrame(pyborrowedref(o))) Base.show(io::IO, x::PyPandasDataFrame) = print(io, pystr(String, x)) Base.show(io::IO, mime::MIME, o::PyPandasDataFrame) = _py_mime_show(io, mime, o) diff --git a/src/PyRef.jl b/src/PyRef.jl index 5d59b55c..4faf219d 100644 --- a/src/PyRef.jl +++ b/src/PyRef.jl @@ -36,7 +36,7 @@ ispyreftype(::Type{PyRef}) = true pyptr(x::PyRef) = x.ptr isnull(x::PyRef) = isnull(pyptr(x)) Base.unsafe_convert(::Type{CPyPtr}, x::PyRef) = pyptr(x) -C.PyObject_TryConvert__initial(o, ::Type{PyRef}) = C.putresult(PyRef, pyborrowedref(o)) +C.PyObject_TryConvert__initial(o, ::Type{PyRef}) = C.putresult(pyborrowedref(o)) PyRef(x) = begin ptr = C.PyObject_From(x) diff --git a/src/PySet.jl b/src/PySet.jl index 9308ea0e..bbabc99d 100644 --- a/src/PySet.jl +++ b/src/PySet.jl @@ -23,7 +23,7 @@ pyptr(x::PySet) = begin ptr end Base.unsafe_convert(::Type{CPyPtr}, x::PySet) = checknull(pyptr(x)) -C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PySet} = C.putresult(T, T(pyborrowedref(o))) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PySet} = C.putresult(T(pyborrowedref(o))) Base.iterate(x::PySet{T}, it::PyRef) where {T} = begin ptr = C.PyIter_Next(it) diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index 514fbb1d..de890378 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -1,7 +1,7 @@ module CPython using Libdl -import ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect, tryconvert, ispyreftype, pyptr, putresult, takeresult, moveresult, CACHE, Python +import ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect, tryconvert, ispyreftype, pyptr, putresult, takeresult, CACHE, Python using Base: @kwdef using UnsafePointers: UnsafePtr @@ -56,6 +56,12 @@ include("newtype.jl") include("juliaerror.jl") include("juliabase.jl") include("juliaraw.jl") +include("juliaany.jl") +include("juliaiterator.jl") +include("juliatype.jl") +include("juliadict.jl") +include("juliaarray.jl") +include("juliavector.jl") include("arg.jl") __init__() = begin @@ -132,7 +138,7 @@ __init__() = begin (Dict, PyMapping_ConvertRule_dict), ]) PyObject_TryConvert_AddRules("julia.ValueBase", [ - (Any, PyJuliaValue_TryConvert_any), + (Any, PyJuliaValue_TryConvert_any, 200), ]) PyObject_TryConvert_AddExtraTypes([ PyIterableABC_Type, diff --git a/src/cpython/arg.jl b/src/cpython/arg.jl index cb506ba6..58041f4e 100644 --- a/src/cpython/arg.jl +++ b/src/cpython/arg.jl @@ -76,7 +76,7 @@ PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,No return PyPtr() end -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Union{Int,Nothing}=nothing, k::Union{String,Nothing}=nothing, d::Union{PyPtr,NODEFAULT}=NODEFAULT()) where {T} = begin +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Union{Int,Nothing}=nothing, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} = begin ro = PyArg_Find(args, kwargs, i, k) if isnull(ro) if k !== nothing @@ -96,14 +96,14 @@ PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Uni PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'") return -1 else - putresult(T, d) + putresult(d) return 0 end else return 0 end end -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}=nothing, d::Union{PyPtr,NODEFAULT}=NODEFAULT()) where {T} = +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} = PyArg_GetArg(T, name, args, PyPtr(), i, k, d) -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr, k::Union{String,Nothing}, d::Union{PyPtr, NODEFAULT}=NODEFAULT()) where {T} = +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr, k::Union{String,Nothing}, d::Union{T, NODEFAULT}=NODEFAULT()) where {T} = PyArg_GetArg(T, name, args, kwargs, nothing, k, d) diff --git a/src/cpython/bool.jl b/src/cpython/bool.jl index 4990352b..d4952e77 100644 --- a/src/cpython/bool.jl +++ b/src/cpython/bool.jl @@ -11,11 +11,11 @@ PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) PyBool_Check(o) = Py_TypeCheckExact(o, PyBool_Type()) -PyBool_TryConvertRule_bool(o, ::Type{T}, ::Type{Bool}) where {T} = +PyBool_TryConvertRule_bool(o, ::Type{Bool}) = if Py_Is(o, Py_True()) - putresult(T, true) + putresult(true) elseif Py_Is(o, Py_False()) - putresult(T, false) + putresult(false) else PyErr_SetString(PyExc_TypeError(), "Expecting a 'bool' but got a '$(PyType_Name(Py_Type(o)))'") -1 diff --git a/src/cpython/bytes.jl b/src/cpython/bytes.jl index d5308949..bbbe3ffb 100644 --- a/src/cpython/bytes.jl +++ b/src/cpython/bytes.jl @@ -27,14 +27,14 @@ PyBytes_AsVector(o, ::Type{T}=UInt8) where {T} = begin copy(Base.unsafe_wrap(Vector{T}, Ptr{T}(ptr[]), len[])) end -PyBytes_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T,X} = begin +PyBytes_TryConvertRule_vector(o, ::Type{Vector{X}}) where {X} = begin v = PyBytes_AsVector(o, X) isempty(v) && PyErr_IsSet() && return -1 - return putresult(T, v) + return putresult(v) end -PyBytes_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin +PyBytes_TryConvertRule_string(o, ::Type{String}) = begin v = PyBytes_AsString(o) isempty(v) && PyErr_IsSet() && return -1 - return putresult(T, v) + return putresult(v) end diff --git a/src/cpython/collections.jl b/src/cpython/collections.jl index 8045c729..e416da52 100644 --- a/src/cpython/collections.jl +++ b/src/cpython/collections.jl @@ -1,3 +1,13 @@ +PyABC_Register(s, t) = begin + r = PyObject_GetAttrString(t, "register") + isnull(r) && return Cint(-1) + u = PyObject_CallNice(r, PyObjectRef(s)) + Py_DecRef(r) + isnull(u) && return Cint(-1) + Py_DecRef(u) + Cint(0) +end + for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, :Sized, :Callable, :Collection, :Sequence, :MutableSequence, :ByteString, :Set, :MutableSet, :Mapping, :MutableMapping, :MappingView, :ItemsView, :KeysView, :ValuesView, @@ -24,7 +34,7 @@ for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, end end -PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = begin +PyIterable_ConvertRule_vector(o, ::Type{S}) where {S<:Vector} = begin it = PyObject_GetIter(o) isnull(it) && return -1 xs = S() @@ -41,14 +51,14 @@ PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{S}) where {T, S<:Vector} = be return -1 else Py_DecRef(it) - return putresult(T, xs) + return putresult(xs) end end end -PyIterable_ConvertRule_vector(o, ::Type{T}, ::Type{Vector}) where {T} = - PyIterable_ConvertRule_vector(o, T, Vector{Python.PyObject}) +PyIterable_ConvertRule_vector(o, ::Type{Vector}) = + PyIterable_ConvertRule_vector(o, Vector{Python.PyObject}) -PyIterable_ConvertRule_set(o, ::Type{T}, ::Type{S}) where {T, S<:Set} = begin +PyIterable_ConvertRule_set(o, ::Type{S}) where {S<:Set} = begin it = PyObject_GetIter(o) isnull(it) && return -1 xs = S() @@ -65,14 +75,14 @@ PyIterable_ConvertRule_set(o, ::Type{T}, ::Type{S}) where {T, S<:Set} = begin return -1 else Py_DecRef(it) - return putresult(T, xs) + return putresult(xs) end end end -PyIterable_ConvertRule_set(o, ::Type{T}, ::Type{Set}) where {T} = +PyIterable_ConvertRule_set(o, ::Type{Set}) = PyIterable_ConvertRule_set(o, T, Set{Python.PyObject}) -PyIterable_ConvertRule_tuple(o, ::Type{T}, ::Type{S}) where {T,S<:Tuple} = begin +PyIterable_ConvertRule_tuple(o, ::Type{S}) where {S<:Tuple} = begin if !(Tuple isa DataType) PyErr_SetString(PyExc_Exception(), "When converting Python 'tuple' to Julia 'Tuple', the destination type must be a 'DataType', i.e. not parametric and not a union. Got '$S'.") return -1 @@ -112,13 +122,13 @@ PyIterable_ConvertRule_tuple(o, ::Type{T}, ::Type{S}) where {T,S<:Tuple} = begin return -1 else Py_DecRef(it) - return putresult(T, S(xs)) + return putresult(S(xs)) end end end -PyIterable_ConvertRule_tuple(o, ::Type{T}, ::Type{Tuple{}}) where {T} = putresult(T, ()) +PyIterable_ConvertRule_tuple(o, ::Type{Tuple{}}) = putresult(()) -PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K,V}}) where {T,K,V} = begin +PyIterable_ConvertRule_pair(o, ::Type{Pair{K,V}}) where {K,V} = begin it = PyObject_GetIter(o) isnull(it) && return -1 # get the first item @@ -158,20 +168,20 @@ PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K,V}}) where {T,K,V} = beg end # done Py_DecRef(it) - putresult(T, Pair{K,V}(k, v)) + putresult(Pair{K,V}(k, v)) end -PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K}}) where {T,K} = - PyIterable_ConvertRule_pair(o, T, Pair{K,Python.PyObject}) -PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair{K,V} where K}) where {T,V} = - PyIterable_ConvertRule_pair(o, T, Pair{Python.PyObject,V}) -PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{Pair}) where {T} = - PyIterable_ConvertRule_pair(o, T, Pair{Python.PyObject,Python.PyObject}) -PyIterable_ConvertRule_pair(o, ::Type{T}, ::Type{S}) where {T,S<:Pair} = begin +PyIterable_ConvertRule_pair(o, ::Type{Pair{K}}) where {K} = + PyIterable_ConvertRule_pair(o, Pair{K,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{Pair{K,V} where K}) where {V} = + PyIterable_ConvertRule_pair(o, Pair{Python.PyObject,V}) +PyIterable_ConvertRule_pair(o, ::Type{Pair}) = + PyIterable_ConvertRule_pair(o, Pair{Python.PyObject,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{S}) where {S<:Pair} = begin PyErr_SetString(PyExc_Exception(), "When converting Python iterable to Julia 'Pair', the destination type cannot be too complicated: the two types must either be fully specified or left unspecified. Got '$S'.") return -1 end -PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{S}) where {T, S<:Dict} = begin +PyMapping_ConvertRule_dict(o, ::Type{S}) where {S<:Dict} = begin it = PyObject_GetIter(o) isnull(it) && return -1 xs = S() @@ -196,13 +206,13 @@ PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{S}) where {T, S<:Dict} = begin return -1 else Py_DecRef(it) - return putresult(T, xs) + return putresult(xs) end end end -PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict{K}}) where {T,K} = - PyMapping_ConvertRule_dict(o, T, Dict{K,Python.PyObject}) -PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict{K,V} where K}) where {T,V} = - PyMapping_ConvertRule_dict(o, T, Dict{Python.PyObject,V}) -PyMapping_ConvertRule_dict(o, ::Type{T}, ::Type{Dict}) where {T} = - PyMapping_ConvertRule_dict(o, T, Dict{Python.PyObject,Python.PyObject}) +PyMapping_ConvertRule_dict(o, ::Type{Dict{K}}) where {K} = + PyMapping_ConvertRule_dict(o, Dict{K,Python.PyObject}) +PyMapping_ConvertRule_dict(o, ::Type{Dict{K,V} where K}) where {V} = + PyMapping_ConvertRule_dict(o, Dict{Python.PyObject,V}) +PyMapping_ConvertRule_dict(o, ::Type{Dict}) = + PyMapping_ConvertRule_dict(o, Dict{Python.PyObject,Python.PyObject}) diff --git a/src/cpython/complex.jl b/src/cpython/complex.jl index a002b326..9d3777d8 100644 --- a/src/cpython/complex.jl +++ b/src/cpython/complex.jl @@ -19,14 +19,14 @@ PyComplex_From(x::Union{Float16,Float32,Float64}) = PyComplex_FromDoubles(x, 0) PyComplex_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_FromDoubles(real(x), imag(x)) # "Complexable" means a 'complex' or anything with a '__complex__' method -PyComplexable_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyComplexable_TryConvertRule_convert(o, ::Type{S}) where {S} = begin x = PyComplex_AsComplex(o) ism1(x) && PyErr_IsSet() && return -1 - putresult(T, convert(S, x)) + putresult(convert(S, x)) end -PyComplexable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyComplexable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin x = PyComplexable_AsComplex(o) ism1(x) && PyErr_IsSet() && return -1 - putresult(T, tryconvert(S, x)) + putresult(tryconvert(S, x)) end diff --git a/src/cpython/ctypes.jl b/src/cpython/ctypes.jl index 258276d5..ffde609d 100644 --- a/src/cpython/ctypes.jl +++ b/src/cpython/ctypes.jl @@ -1,7 +1,7 @@ struct PySimpleCData_TryConvert_value{R,tr} end -(::PySimpleCData_TryConvert_value{R,tr})(o, ::Type{T}, ::Type{S}) where {T,S,R,tr} = begin +(::PySimpleCData_TryConvert_value{R,tr})(o, ::Type{S}) where {S,R,tr} = begin ptr = PySimpleObject_GetValue(o, Ptr{R}) val = unsafe_load(ptr) - putresult(T, tr ? tryconvert(S, val) : convert(S, val)) + putresult(tr ? tryconvert(S, val) : convert(S, val)) end diff --git a/src/cpython/float.jl b/src/cpython/float.jl index 0ecd8b14..37e0315f 100644 --- a/src/cpython/float.jl +++ b/src/cpython/float.jl @@ -11,14 +11,14 @@ PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) # "Floatable" means a 'float' or anything with a '__float__' method -PyFloatable_TryConvertRule_convert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyFloatable_TryConvertRule_convert(o, ::Type{S}) where {S} = begin x = PyFloat_AsDouble(o) ism1(x) && PyErr_IsSet() && return -1 - putresult(T, convert(S, x)) + putresult(convert(S, x)) end -PyFloatable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyFloatable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin x = PyFloat_AsDouble(o) ism1(x) && PyErr_IsSet() && return -1 - putresult(T, tryconvert(S, x)) + putresult(tryconvert(S, x)) end diff --git a/src/cpython/int.jl b/src/cpython/int.jl index b170e267..abb65931 100644 --- a/src/cpython/int.jl +++ b/src/cpython/int.jl @@ -28,12 +28,12 @@ PyLong_From(x::Integer) = begin end # "Longable" means an 'int' or anything with an '__int__' method. -PyLongable_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} = begin +PyLongable_TryConvertRule_integer(o, ::Type{S}) where {S<:Integer} = begin # first try to convert to Clonglong (or Culonglong if unsigned) x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) if !ism1(x) || !PyErr_IsSet() # success - return putresult(T, tryconvert(S, x)) + return putresult(tryconvert(S, x)) elseif PyErr_IsSet(PyExc_OverflowError()) # overflows Clonglong or Culonglong PyErr_Clear() @@ -44,12 +44,12 @@ PyLongable_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} # try converting to String then BigInt then S so = PyObject_Str(o) isnull(so) && return -1 - r = PyUnicode_TryConvertRule_string(so, String, String) + s = PyUnicode_AsString(so) Py_DecRef(so) - r == 1 || return r - y = tryparse(BigInt, takeresult(String)) + isempty(s) && PyErr_IsSet() && return -1 + y = tryparse(BigInt, s) y === nothing && (PyErr_SetString(PyExc_ValueError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia 'BigInt' because its string representation cannot be parsed as an integer"); return -1) - return putresult(T, tryconvert(S, y)) + return putresult(tryconvert(S, y)) end else # other error @@ -57,7 +57,7 @@ PyLongable_TryConvertRule_integer(o, ::Type{T}, ::Type{S}) where {T, S<:Integer} end end -PyLongable_TryConvertRule_tryconvert(o, ::Type{T}, ::Type{S}) where {T,S} = begin - r = PyLong_TryConvertRule_integer(o, Integer, Integer) - r == 1 ? putresult(T, tryconvert(S, takeresult(Integer))) : r +PyLongable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin + r = PyLongable_TryConvertRule_integer(o, Integer) + r == 1 ? putresult(tryconvert(S, takeresult(Integer))) : r end diff --git a/src/cpython/juliabase.jl b/src/cpython/juliabase.jl index 401aabc4..f49fcb83 100644 --- a/src/cpython/juliabase.jl +++ b/src/cpython/juliabase.jl @@ -55,7 +55,7 @@ PyJuliaBaseValue_Type() = begin ptr end -PyJuliaValue_Check(o) = Py_TypeCheck(o, PyJuliaBaseValue_Type()) != 0 +PyJuliaValue_Check(o) = Py_TypeCheck(o, PyJuliaBaseValue_Type()) PyJuliaValue_GetValue(__o) = begin _o = Base.cconvert(PyPtr, __o) @@ -76,6 +76,12 @@ PyJuliaValue_SetValue(__o, v) = begin end PyJuliaValue_New(t, v) = begin + if isnull(t) + if !PyErr_IsSet() + PyErr_SetString(PyExc_Exception(), "Got NULL type with no error set") + end + return PyPtr() + end bt = PyJuliaBaseValue_Type() isnull(bt) && return PyPtr() PyType_IsSubtype(t, bt) != 0 || (PyErr_SetString(PyExc_TypeError(), "Expecting a subtype of 'julia.ValueBase'"); return PyPtr()) @@ -92,7 +98,7 @@ PyJuliaBaseValue_New(v) = begin PyJuliaValue_New(t, v) end -PyJuliaValue_TryConvert_any(o, ::Type{T}, ::Type{S}) where {T,S} = begin +PyJuliaValue_TryConvert_any(o, ::Type{S}) where {S} = begin x = PyJuliaValue_GetValue(o) - putresult(T, tryconvert(S, x)) + putresult(tryconvert(S, x)) end diff --git a/src/cpython/juliaerror.jl b/src/cpython/juliaerror.jl index 35f233dc..dd346c09 100644 --- a/src/cpython/juliaerror.jl +++ b/src/cpython/juliaerror.jl @@ -6,6 +6,7 @@ PyExc_JuliaError() = begin t = fill(PyType_Create(c, name = "julia.Error", base = PyExc_BaseException(), + str = pyjlerr_str, )) ptr = PyPtr(pointer(t)) err = PyType_Ready(ptr) @@ -25,7 +26,7 @@ PyErr_SetJuliaError(err, bt=nothing) = begin if bt === nothing bt = catch_backtrace() end - v = PyJuliaBaseValue_New((err, bt)) + v = PyJuliaValue_From((err, bt)) if isnull(v) PyErr_Clear() PyErr_SetString(t, string(err)) @@ -35,3 +36,34 @@ PyErr_SetJuliaError(err, bt=nothing) = begin end end end + +PyErr_SetStringFromJuliaError(t, err) = begin + io = IOBuffer() + showerror(io, err) + msg = String(take!(io)) + PyErr_SetString(t, "Julia: $msg") +end + +pyjlerr_str(xo::PyPtr) = begin + args = PyObject_GetAttrString(xo, "args") + isnull(args) && return PyPtr() + r = PyObject_TryConvert(args, Tuple{Union{String, Tuple}}) + r == -1 && (Py_DecRef(args); return PyPtr()) + r == 0 && @goto fallback + (x,) = takeresult(Tuple{Union{String, Tuple}}) + if x isa String + Py_DecRef(args) + return PyUnicode_From(x) + elseif x isa Tuple{Exception, Any} + Py_DecRef(args) + io = IOBuffer() + showerror(io, x[1]) + msg = String(take!(io)) + return PyUnicode_From(msg) + end + + @label fallback + so = PyObject_Str(args) + Py_DecRef(args) + return so +end diff --git a/src/cpython/juliaraw.jl b/src/cpython/juliaraw.jl index d092340b..b26158fd 100644 --- a/src/cpython/juliaraw.jl +++ b/src/cpython/juliaraw.jl @@ -1,7 +1,4 @@ -JLRAWTYPES = Dict{Type, PyPtr}() - const PyJuliaRawValue_Type__ref = Ref(PyPtr()) - PyJuliaRawValue_Type() = begin ptr = PyJuliaRawValue_Type__ref[] if isnull(ptr) @@ -34,12 +31,14 @@ PyJuliaRawValue_Type() = begin ptr end +PyJuliaRawValue_New(x) = PyJuliaValue_New(PyJuliaRawValue_Type(), x) + pyjlraw_repr(xo::PyPtr) = try x = PyJuliaValue_GetValue(xo) s = "" PyUnicode_From(s) catch err - PyErr_SetJuliaError(err, catch_backtrace()) + PyErr_SetJuliaError(err) return PyPtr() end @@ -48,10 +47,13 @@ pyjlraw_str(xo::PyPtr) = try s = string(x) PyUnicode_From(s) catch err - PyErr_SetJuliaError(err, catch_backtrace()) + PyErr_SetJuliaError(err) return PyPtr() end +pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x)-1))) +pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) + pyjlraw_getattro(xo::PyPtr, ko::PyPtr) = begin # Try generic lookup first ro = PyObject_GenericGetAttr(xo, ko) @@ -64,6 +66,7 @@ pyjlraw_getattro(xo::PyPtr, ko::PyPtr) = begin x = PyJuliaValue_GetValue(xo) k = PyUnicode_AsString(ko) isempty(k) && PyErr_IsSet() && return PyPtr() + k = pyjl_attr_py2jl(k) try v = getproperty(x, Symbol(k)) PyJuliaRawValue_New(v) @@ -85,6 +88,7 @@ pyjlraw_setattro(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin x = PyJuliaValue_GetValue(xo) k = PyUnicode_AsString(ko) isempty(k) && PyErr_IsSet() && return Cint(-1) + k = pyjl_attr_py2jl(k) ism1(PyObject_Convert(vo, Any)) && return Cint(-1) v = takeresult(Any) try @@ -104,14 +108,14 @@ pyjlraw_dir(xo::PyPtr, _::PyPtr) = begin isnull(ro) && return PyPtr() x = PyJuliaValue_GetValue(xo) ks = try - collect(map(string, propertynames(x)))::Vector{String} + collect(map(string, propertynames(x))) catch err Py_DecRef(ro) PyErr_SetJuliaError(err) return PyPtr() end for k in ks - ko = PyUnicode_From(k) + ko = PyUnicode_From(pyjl_attr_jl2py(k)) isnull(ko) && (Py_DecRef(ro); return PyPtr()) err = PyList_Append(ro, ko) Py_DecRef(ko) @@ -200,9 +204,3 @@ pyjlraw_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin end end end - -PyJuliaRawValue_New(x) = begin - t = PyJuliaRawValue_Type() - isnull(t) && return PyPtr() - PyJuliaValue_New(t, x) -end diff --git a/src/cpython/newtype.jl b/src/cpython/newtype.jl index f6e9fe08..74571038 100644 --- a/src/cpython/newtype.jl +++ b/src/cpython/newtype.jl @@ -43,7 +43,20 @@ PyMappingMethods_Create(c, x::Dict) = PyMappingMethods_Create(c; x...) PyMappingMethods_Create(c, x::NamedTuple) = PyMappingMethods_Create(c; x...) PySequenceMethods_Create(c, x::PySequenceMethods) = x -PySequenceMethods_Create(c; opts...) = C.PySequenceMethods(; [k => (v isa Ptr ? v : v isa Base.CFunction ? cacheptr!(c, v) : error()) for (k,v) in pairs(opts)]...) +PySequenceMethods_Create(c; + length=C_NULL, concat=C_NULL, repeat=C_NULL, item=C_NULL, ass_item=C_NULL, + contains=C_NULL, inplace_concat=C_NULL, inplace_repeat=C_NULL, +) = + PySequenceMethods( + length = @cachefuncptr!(c, length, Py_ssize_t, (PyPtr,)), + concat = @cachefuncptr!(c, concat, PyPtr, (PyPtr, PyPtr)), + repeat = @cachefuncptr!(c, repeat, PyPtr, (PyPtr, Py_ssize_t)), + item = @cachefuncptr!(c, item, PyPtr, (PyPtr, Py_ssize_t)), + ass_item = @cachefuncptr!(c, ass_item, Cint, (PyPtr, Py_ssize_t, PyPtr)), + contains = @cachefuncptr!(c, contains, Cint, (PyPtr, PyPtr)), + inplace_concat = @cachefuncptr!(c, inplace_concat, PyPtr, (PyPtr, PyPtr)), + inplace_repeat = @cachefuncptr!(c, inplace_repeat, PyPtr, (PyPtr, Py_ssize_t)), + ) PySequenceMethods_Create(c, x::Dict) = PySequenceMethods_Create(c; x...) PySequenceMethods_Create(c, x::NamedTuple) = PySequenceMethods_Create(c; x...) @@ -60,7 +73,7 @@ PyMethodDef_Create(c, x::PyMethodDef) = x PyMethodDef_Create(c; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = PyMethodDef( name = cachestrptr!(c, name), - meth = iszero(flags & Py_METH_VARARGS) ? @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr)) : @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr, PyPtr)), + meth = iszero(flags & Py_METH_KEYWORDS) ? @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr)) : @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr, PyPtr)), flags = flags, doc = cachestrptr!(c, doc), ) @@ -83,7 +96,7 @@ PyType_Create(c; type=C_NULL, name, as_number=C_NULL, as_mapping=C_NULL, as_sequence=C_NULL, as_buffer=C_NULL, methods=C_NULL, getset=C_NULL, dealloc=C_NULL, getattr=C_NULL, setattr=C_NULL, repr=C_NULL, hash=C_NULL, call=C_NULL, str=C_NULL, getattro=C_NULL, - setattro=C_NULL, doc=C_NULL, iter=C_NULL, iternext=C_NULL, opts... + setattro=C_NULL, doc=C_NULL, iter=C_NULL, iternext=C_NULL, richcompare=C_NULL, opts... ) = begin type = cacheptr!(c, type) name = cachestrptr!(c, name) @@ -114,12 +127,13 @@ PyType_Create(c; setattro = @cachefuncptr!(c, setattro, Cint, (PyPtr, PyPtr, PyPtr)) doc = cachestrptr!(c, doc) iter = @cachefuncptr!(c, iter, PyPtr, (PyPtr,)) - iternext = @cachefuncptr!(c, iter, PyPtr, (PyPtr,)) + iternext = @cachefuncptr!(c, iternext, PyPtr, (PyPtr,)) + richcompare = @cachefuncptr!(c, richcompare, PyPtr, (PyPtr, PyPtr, Cint)) PyTypeObject(; ob_base=PyVarObject(ob_base=PyObject(type=type)), name=name, as_number=as_number, as_mapping=as_mapping, as_sequence=as_sequence, as_buffer=as_buffer, methods=methods, getset=getset, dealloc=dealloc, getattr=getattr, setattr=setattr, repr=repr, hash=hash, call=call, str=str, getattro=getattro, setattro=setattro, - iter=iter, iternext=iternext, opts... + iter=iter, iternext=iternext, richcompare=richcompare, opts... ) end diff --git a/src/cpython/none.jl b/src/cpython/none.jl index c5760202..f3da54df 100644 --- a/src/cpython/none.jl +++ b/src/cpython/none.jl @@ -1,3 +1,10 @@ +const Py_NotImplemented__ref = Ref(PyPtr()) +Py_NotImplemented() = pyglobal(Py_NotImplemented__ref, :_Py_NotImplementedStruct) + +PyNotImplemented_Check(o) = Py_Is(o, Py_NotImplemented()) + +PyNotImplemented_New() = (o=Py_NotImplemented(); Py_IncRef(o); o) + const Py_None__ref = Ref(PyPtr()) Py_None() = pyglobal(Py_None__ref, :_Py_NoneStruct) @@ -5,5 +12,5 @@ PyNone_Check(o) = Py_Is(o, Py_None()) PyNone_New() = (o=Py_None(); Py_IncRef(o); o) -PyNone_TryConvertRule_nothing(o, ::Type{T}, ::Type{Nothing}) where {T} = putresult(T, nothing) -PyNone_TryConvertRule_missing(o, ::Type{T}, ::Type{Missing}) where {T} = putresult(T, missing) +PyNone_TryConvertRule_nothing(o, ::Type{Nothing}) = putresult(nothing) +PyNone_TryConvertRule_missing(o, ::Type{Missing}) = putresult(missing) diff --git a/src/cpython/numpy.jl b/src/cpython/numpy.jl index dc74ee04..c2637951 100644 --- a/src/cpython/numpy.jl +++ b/src/cpython/numpy.jl @@ -1,6 +1,6 @@ struct PyNumpySimpleData_TryConvert_value{R,tr} end -(::PyNumpySimpleData_TryConvert_value{R,tr})(o, ::Type{T}, ::Type{S}) where {T,S,R,tr} = begin +(::PyNumpySimpleData_TryConvert_value{R,tr})(o, ::Type{S}) where {S,R,tr} = begin val = PySimpleObject_GetValue(o, R) - putresult(T, tr ? tryconvert(S, val) : convert(S, val)) + putresult(tr ? tryconvert(S, val) : convert(S, val)) end diff --git a/src/cpython/object.jl b/src/cpython/object.jl index 4d3679fb..48188192 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -43,20 +43,18 @@ PyObject_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_From(x) PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) PyObject_From(x::Char) = PyUnicode_From(string(x)) PyObject_From(x::Tuple) = PyTuple_From(x) -PyObject_From(x::AbstractVector) = PyList_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED -PyObject_From(x::AbstractDict) = PyDict_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED -PyObject_From(x::AbstractSet) = PySet_From(x) # REMOVE WHEN PYJULIA IMPLEMENTED PyObject_From(x::AbstractRange{<:Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyRange_From(x) -PyObject_From(x::T) where {T} = - if ispyreftype(T) +PyObject_From(x) = + if ispyreftype(typeof(x)) GC.@preserve x begin ptr = pyptr(x) Py_IncRef(ptr) ptr end else - PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") - PyPtr() + PyJuliaValue_From(x) + # PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") + # PyPtr() end function PyObject_CallArgs(f, args::Tuple, kwargs::Union{Nothing,NamedTuple,Base.Iterators.Pairs{Symbol},Base.Iterators.Pairs{Union{}}}=nothing) @@ -82,16 +80,12 @@ end PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) -const ERRPTR = Ptr{Cvoid}(1) - """ -Mapping of Julia types to mappings of Python types to compiled functions implementing the conversion. - -That is, `ccall(TRYCONVERT_COMPILED_RULES[T][Py_Type(o)], Int, (PyPtr,), o)` attempts to convert `o` to a `T`. -On success, this returns `1` and the result can be obtained with `takeresult(T)`. -Otherwise returns `0` if no conversion was possible, or `-1` on error. +Mapping of Julia types to mappings of Python types to vectors of compiled functions implementing the conversion. """ -const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Ptr{Cvoid}}}() +const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Vector{Ptr{Cvoid}}}}() + +const TRYCONVERT_COMPILED_RULES_CACHE = Dict{Any, PyPtr}() """ Mapping of type names to lists of rules. @@ -107,7 +101,7 @@ The rule is applied only when the `type` has non-trivial intersection with the t Finally `impl` is the function implementing the conversion. Its signature is `(o, T, S)` where `o` is the `PyPtr` object to convert, `T` is the desired type, and `S = typeintersect(T, type)`. -On success it returns `putresult(T, x)` where `x` is the converted value (this stores the result and returns `1`). +On success it returns `putresult(x)` where `x::T` is the converted value (this stores the result and returns `1`). If conversion was not possible, returns `0` (indicating we move on to the next rule in the list). On error, returns `-1`. """ @@ -121,7 +115,8 @@ Can also return NULL. Without an error set, this indicates the type is not loade const TRYCONVERT_EXTRATYPES = Vector{Any}() @generated PyObject_TryConvert_CompiledRules(::Type{T}) where {T} = - get!(Dict{PyPtr, Ptr{Cvoid}}, TRYCONVERT_COMPILED_RULES, T) + get!(Dict{PyPtr, Vector{Ptr{Cvoid}}}, TRYCONVERT_COMPILED_RULES, T) + PyObject_TryConvert_Rules(n::String) = get!(Vector{Tuple{Int, Type, Function}}, TRYCONVERT_RULES, n) PyObject_TryConvert_AddRule(n::String, T, rule, priority=0) = @@ -151,11 +146,11 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin basemros = Vector{PyPtr}[tmro] for xtf in TRYCONVERT_EXTRATYPES xt = xtf() - isnull(xt) && (PyErr_IsSet() ? (return ERRPTR) : continue) + isnull(xt) && (PyErr_IsSet() ? (return PYERR()) : continue) xb = PyPtr() for b in tmro r = PyObject_IsSubclass(b, xt) - ism1(r) && return ERRPTR + ism1(r) && return PYERR() r != 0 && (xb = b) end if xb != PyPtr() @@ -223,40 +218,47 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin @debug "PYTHON CONVERSION FOR '$(PyType_FullName(t))' to '$T'" basetypes=map(PyType_FullName, basetypes) alltypes=allnames rules=rules - ### STAGE 3: Define and compile a function implementing these rules. + ### STAGE 3: Define and compile functions implementing these rules. - # make the function implementing these rules - rulefunc = @eval (o::PyPtr) -> begin - $((:(r = $rule(o, $T, $S)::Int; r == 0 || return r) for (S,rule) in rules)...) - return 0 + map(rules) do x + get!(TRYCONVERT_COMPILED_RULES_CACHE, x) do + S, rule = x + # make the function implementing the rule + rulefunc = @eval (o::PyPtr) -> $rule(o, $S) + # compile it + crulefunc = @cfunction($rulefunc, Int, (PyPtr,)) + # cache it + push!(CACHE, crulefunc) + # get the function pointer + Base.unsafe_convert(Ptr{Cvoid}, crulefunc) + end end - # compile it - rulecfunc = @cfunction($rulefunc, Int, (PyPtr,)) - push!(CACHE, rulecfunc) - Base.unsafe_convert(Ptr{Cvoid}, rulecfunc) end +const TRYCONVERT_ERR_ARRAY = Ptr{Cvoid}[] + PyObject_TryConvert(o::PyPtr, ::Type{T}) where {T} = begin # First try based only on the type T - # Used mainly by wrapper types. + # Used mainly by wrapper types for immediate conversion. r = PyObject_TryConvert__initial(o, T) :: Int r == 0 || return r # Try to find an appropriate conversion based on `T` and the type of `o`. rules = PyObject_TryConvert_CompiledRules(T) t = Py_Type(o) - rule = get(rules, t, ERRPTR) - if rule == ERRPTR - rule = PyObject_TryConvert_CompileRule(T, t) - rule == ERRPTR && return -1 - rules[t] = rule + crules = get(rules, t, TRYCONVERT_ERR_ARRAY) + if crules === TRYCONVERT_ERR_ARRAY + crules = PyObject_TryConvert_CompileRule(T, t) + crules === PYERR() && return -1 + rules[t] = crules end - if !isnull(rule) - r = ccall(rule, Int, (PyPtr,), o) + for crule in crules + r = ccall(crule, Int, (PyPtr,), o) r == 0 || return r end - 0 + # Failed to convert + return 0 end PyObject_TryConvert(o, ::Type{T}) where {T} = GC.@preserve o PyObject_TryConvert(Base.unsafe_convert(PyPtr, o), T) diff --git a/src/cpython/range.jl b/src/cpython/range.jl index c2686644..9cad1f1d 100644 --- a/src/cpython/range.jl +++ b/src/cpython/range.jl @@ -31,7 +31,7 @@ PyRange_From(r::AbstractRange{<:Integer}) = steptype(::Type{<:(StepRange{A,B} where {A})}) where {B} = B steptype(::Type{<:StepRange}) = Any -PyRange_TryConvertRule_steprange(o, ::Type{T}, ::Type{S}) where {T,S<:StepRange} = begin +PyRange_TryConvertRule_steprange(o, ::Type{S}) where {S<:StepRange} = begin A = _typeintersect(Integer, eltype(S)) B = _typeintersect(Integer, steptype(S)) # get start @@ -57,10 +57,10 @@ PyRange_TryConvertRule_steprange(o, ::Type{T}, ::Type{S}) where {T,S<:StepRange} c = takeresult(A) # success a′, c′ = promote(a, c - oftype(c, sign(b))) - putresult(T, tryconvert(S, StepRange(a′, b, c′))) + putresult(tryconvert(S, StepRange(a′, b, c′))) end -PyRange_TryConvertRule_unitrange(o, ::Type{T}, ::Type{S}) where {T,S<:UnitRange} = begin +PyRange_TryConvertRule_unitrange(o, ::Type{S}) where {S<:UnitRange} = begin A = _typeintersect(Integer, eltype(S)) # get step bo = PyObject_GetAttrString(o, "step") @@ -86,5 +86,5 @@ PyRange_TryConvertRule_unitrange(o, ::Type{T}, ::Type{S}) where {T,S<:UnitRange} c = takeresult(A) # success a′, c′ = promote(a, c - oftype(c, 1)) - putresult(T, tryconvert(S, UnitRange(a′, c′))) + putresult(tryconvert(S, UnitRange(a′, c′))) end diff --git a/src/cpython/str.jl b/src/cpython/str.jl index 26a77982..de9c139a 100644 --- a/src/cpython/str.jl +++ b/src/cpython/str.jl @@ -27,27 +27,27 @@ PyUnicode_AsVector(o, ::Type{T}=UInt8) where {T} = begin r end -PyUnicode_TryConvertRule_string(o, ::Type{T}, ::Type{String}) where {T} = begin +PyUnicode_TryConvertRule_string(o, ::Type{String}) = begin r = PyUnicode_AsString(o) isempty(r) && PyErr_IsSet() && return -1 - putresult(T, r) + putresult(r) end -PyUnicode_TryConvertRule_vector(o, ::Type{T}, ::Type{Vector{X}}) where {T, X} = begin +PyUnicode_TryConvertRule_vector(o, ::Type{Vector{X}}) where {X} = begin r = PyUnicode_AsVector(o, X) isempty(r) && PyErr_IsSet() && return -1 - putresult(T, r) + putresult(r) end -PyUnicode_TryConvertRule_symbol(o, ::Type{T}, ::Type{Symbol}) where {T} = begin +PyUnicode_TryConvertRule_symbol(o, ::Type{Symbol}) = begin r = PyUnicode_AsString(o) isempty(r) && PyErr_IsSet() && return -1 - putresult(T, Symbol(r)) + putresult(Symbol(r)) end -PyUnicode_TryConvertRule_char(o, ::Type{T}, ::Type{Char}) where {T} = begin +PyUnicode_TryConvertRule_char(o, ::Type{Char}) = begin r = PyUnicode_AsString(o) isempty(r) && PyErr_IsSet() && return -1 length(r) == 1 || return 0 - putresult(T, first(r)) + putresult(first(r)) end diff --git a/src/cpython/type.jl b/src/cpython/type.jl index 762c1f71..39125af0 100644 --- a/src/cpython/type.jl +++ b/src/cpython/type.jl @@ -2,7 +2,7 @@ @cdef :PyType_Ready Cint (PyPtr,) Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] -Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) +Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) != 0 Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) @@ -24,17 +24,15 @@ PyType_FullName(o) = begin # get __module__ mo = PyObject_GetAttrString(o, "__module__") isnull(mo) && return PYERR() - r = PyUnicode_TryConvertRule_string(mo, String, String) + m = PyUnicode_AsString(mo) Py_DecRef(mo) - r == 1 || return PYERR() - m = takeresult(String) + isempty(m) && PyErr_IsSet() && return PYERR() # get __qualname__ no = PyObject_GetAttrString(o, "__qualname__") isnull(no) && return PYERR() - r = PyUnicode_TryConvertRule_string(no, String, String) + n = PyUnicode_AsString(no) Py_DecRef(no) - r == 1 || return PYERR() - n = takeresult(String) + isempty(n) && PyErr_IsSet() && return PYERR() # done "$m.$n" end diff --git a/src/julia.jl b/src/julia.jl index 1fc4287e..f8c2c71b 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -13,6 +13,15 @@ pyjlraw(::Type{T}, x) where {T} = checknullconvert(T, C.PyJuliaRawValue_New(x)) pyjlraw(x) = pyjlraw(PyObject, x) export pyjlraw +""" + pyjl([T=PyObject,] x) + +Wrap `x` as a Python `julia.AnyValue` (or subclass) object. +""" +pyjl(::Type{T}, x) where {T} = checknullconvert(T, C.PyJuliaValue_From(x)) +pyjl(x) = pyjl(PyObject, x) +export pyjl + """ pyjlgetvalue() """ diff --git a/src/utils.jl b/src/utils.jl index 5de76834..4b4d1dc0 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -76,19 +76,10 @@ struct NOTIMPLEMENTED end # Somewhere to stash results const RESULT = Ref{Any}(nothing) -_putresult(::Type{T}, x::T) where {T} = (RESULT[] = x) -takeresult(::Type{T}) where {T} = (r = RESULT[]::T; RESULT[] = nothing; r) - -const RESULT_INT = Ref{Int}() -_putresult(::Type{Int}, x::Int) = (RESULT_INT[] = x) -takeresult(::Type{Int}) = RESULT_INT[] - -putresult(::Type{T}, x::T) where {T} = (_putresult(T, x); 1) -putresult(::Type{T}, x::PYERR) where {T} = -1 -putresult(::Type{T}, x::NOTIMPLEMENTED) where {T} = 0 - -moveresult(::Type{T}, ::Type{T}) where {T} = 1 -moveresult(::Type{S}, ::Type{T}) where {S,T} = putresult(T, takeresult(S)) +putresult(x) = (RESULT[] = x; 1) +putresult(x::PYERR) = -1 +putresult(x::NOTIMPLEMENTED) = 0 +takeresult(::Type{T}=Any) where {T} = (r = RESULT[]::T; RESULT[] = nothing; r) tryconvert(::Type{T}, x::PYERR) where {T} = PYERR() tryconvert(::Type{T}, x::NOTIMPLEMENTED) where {T} = NOTIMPLEMENTED() @@ -100,7 +91,7 @@ tryconvert(::Type{T}, x) where {T} = NOTIMPLEMENTED() end -CTryConvertRule_wrapref(o, ::Type{T}, ::Type{S}) where {T,S} = putresult(T, S(C.PyObjectRef(o))) +CTryConvertRule_wrapref(o, ::Type{S}) where {S} = putresult(S(C.PyObjectRef(o))) @generated _typeintersect(::Type{T1}, ::Type{T2}) where {T1,T2} = typeintersect(T1, T2) From a32645786e7fd5f3d6c1241ac6d83f6759823c6c Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Tue, 22 Dec 2020 18:35:15 +0000 Subject: [PATCH 12/14] dev --- juliapy/julia/__init__.py | 30 ++-- src/cpython/juliaany.jl | 316 +++++++++++++++++++++++++++++++++++ src/cpython/juliaarray.jl | 184 ++++++++++++++++++++ src/cpython/juliadict.jl | 85 ++++++++++ src/cpython/juliaiterator.jl | 57 +++++++ src/cpython/juliatype.jl | 56 +++++++ src/cpython/juliavector.jl | 50 ++++++ 7 files changed, 758 insertions(+), 20 deletions(-) create mode 100644 src/cpython/juliaany.jl create mode 100644 src/cpython/juliaarray.jl create mode 100644 src/cpython/juliadict.jl create mode 100644 src/cpython/juliaiterator.jl create mode 100644 src/cpython/juliatype.jl create mode 100644 src/cpython/juliavector.jl diff --git a/juliapy/julia/__init__.py b/juliapy/julia/__init__.py index d8a10217..543407d3 100644 --- a/juliapy/julia/__init__.py +++ b/juliapy/julia/__init__.py @@ -1,6 +1,6 @@ -_CONFIG = dict() +CONFIG = dict() -def _init_(): +def init(): import os, os.path, sys, ctypes as c, types, shutil, subprocess libpath = os.environ.get('JULIAPY_LIB') if libpath is None: @@ -12,12 +12,12 @@ def _init_(): else: if not os.path.isfile(exepath): raise Exception('JULIAPY_EXE=%s does not exist' % repr(exepath)) - _CONFIG['exepath'] = exepath + CONFIG['exepath'] = exepath libpath = subprocess.run([exepath, '-e', 'import Libdl; print(abspath(Libdl.dlpath("libjulia")))'], stdout=(subprocess.PIPE)).stdout.decode('utf8') else: if not os.path.isfile(libpath): raise Exception('JULIAPY_LIB=%s does not exist' % repr(libpath)) - _CONFIG['libpath'] = libpath + CONFIG['libpath'] = libpath try: d = os.getcwd() os.chdir(os.path.dirname(libpath)) @@ -25,7 +25,7 @@ def _init_(): finally: os.chdir(d) - _CONFIG['lib'] = lib + CONFIG['lib'] = lib lib.jl_init__threading.argtypes = [] lib.jl_init__threading.restype = None lib.jl_init__threading() @@ -37,7 +37,7 @@ def _init_(): ENV["PYTHONJL_LIBPTR"] = "{}" import Python Python.with_gil() do - Python.pyimport("sys").modules["julia"].Main = Python.pyjlraw(Main) + Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main) end catch err @error "Error loading Python.jl" err=err @@ -47,22 +47,12 @@ def _init_(): if res is None: raise Exception('Python.jl did not start properly. Ensure that the Python package is installed in Julia.') - class Wrapper(types.ModuleType): - - def __getattr__(self, k): - return getattr(self.Main, k) - - def __dir__(self): - return super().__dir__() + self.Main.__dir__() - - sys.modules['julia'].__class__ = Wrapper - -_init_() -del _init_ +init() +del init Core = Main.Core Base = Main.Base Python = Main.Python -def _import(*names): - Main.eval(Base.Meta.parse('import ' + ', '.join(names))) +def newmodule(name): + return Base.Module(Base.Symbol(name)) diff --git a/src/cpython/juliaany.jl b/src/cpython/juliaany.jl new file mode 100644 index 00000000..c51451ee --- /dev/null +++ b/src/cpython/juliaany.jl @@ -0,0 +1,316 @@ +const PyJuliaAnyValue_Type__ref = Ref(PyPtr()) +PyJuliaAnyValue_Type() = begin + ptr = PyJuliaAnyValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaBaseValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.AnyValue", + base = base, + repr = pyjlany_repr, + str = pyjlany_str, + getattro = pyjlany_getattro, + setattro = pyjlany_setattro, + call = pyjlany_call, + iter = pyjlany_iter, + richcompare = pyjlany_richcompare, + as_mapping = ( + length = pyjlany_length, + subscript = pyjlany_getitem, + ass_subscript = pyjlany_setitem, + ), + as_sequence = ( + contains = pyjlany_contains, + ), + methods = [ + (name="__dir__", flags=Py_METH_NOARGS, meth=pyjlany_dir), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaAnyValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaAnyValue_New(x) = PyJuliaValue_New(PyJuliaAnyValue_Type(), x) +PyJuliaValue_From(x) = PyJuliaAnyValue_New(x) + +pyjlany_repr(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = "" + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlany_str(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = string(x) + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + return PyPtr() +end + +pyjlany_getattro(xo::PyPtr, ko::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericGetAttr(xo, ko) + if isnull(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to get the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return PyPtr() + k = pyjl_attr_py2jl(k) + try + v = getproperty(x, Symbol(k)) + PyObject_From(v) + catch err + if (err isa UndefVarError && err.var === Symbol(k)) || (err isa ErrorException && occursin("has no field", err.msg)) + PyErr_SetStringFromJuliaError(PyExc_AttributeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +propertytype(x, k) = + propertiesarefields(typeof(x)) && hasfield(typeof(x), k) ? fieldtype(typeof(x), k) : Any +@generated propertiesarefields(::Type{T}) where {T} = + which(getproperty, Tuple{T,Symbol}) == which(getproperty, Tuple{Any,Symbol}) + +pyjlany_setattro(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericSetAttr(xo, ko, vo) + if ism1(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + if isnull(vo) + PyErr_SetString(PyExc_TypeError(), "attribute deletion not supported") + return Cint(-1) + end + # Now try to set the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return Cint(-1) + k = pyjl_attr_py2jl(k) + try + V = propertytype(x, Symbol(k)) + ism1(PyObject_Convert(vo, V)) && return Cint(-1) + v = takeresult(V) + setproperty!(x, Symbol(k), v) + Cint(0) + catch err + if (err isa UndefVarError && err.var === Symbol(k)) || (err isa ErrorException && occursin("has no field", err.msg)) + PyErr_SetStringFromJuliaError(PyExc_AttributeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjl_dir(x) = propertynames(x) +pyjl_dir(x::Module) = begin + r = Symbol[] + append!(r, names(x, all=true, imported=true)) + for m in ccall(:jl_module_usings, Any, (Any,), x)::Vector + append!(r, names(m)) + end + r +end + +pyjlany_dir(xo::PyPtr, _::PyPtr) = begin + fo = PyObject_GetAttrString(PyJuliaBaseValue_Type(), "__dir__") + isnull(fo) && return PyPtr() + ro = PyObject_CallNice(fo, PyObjectRef(xo)) + Py_DecRef(fo) + isnull(ro) && return PyPtr() + x = PyJuliaValue_GetValue(xo) + ks = try + collect(map(string, pyjl_dir(x))) + catch err + Py_DecRef(ro) + PyErr_SetJuliaError(err) + return PyPtr() + end + for k in ks + ko = PyUnicode_From(pyjl_attr_jl2py(k)) + isnull(ko) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, ko) + Py_DecRef(ko) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + return ro +end + +pyjlany_call(fo::PyPtr, argso::PyPtr, kwargso::PyPtr) = begin + f = PyJuliaValue_GetValue(fo) + if isnull(argso) + args = Vector{Any}() + else + ism1(PyObject_Convert(argso, Vector{Any})) && return PyPtr() + args = takeresult(Vector{Any}) + end + if isnull(kwargso) + kwargs = Dict{Symbol, Any}() + else + ism1(PyObject_Convert(kwargso, Dict{Symbol, Any})) && return PyPtr() + kwargs = takeresult(Dict{Symbol, Any}) + end + try + x = f(args...; kwargs...) + PyObject_From(x) + catch err + if err isa MethodError && err.f === f + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +pyjlany_length(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + Py_ssize_t(length(x)) +catch err + if err isa MethodError && err.f === length + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Py_ssize_t(-1) +end + +@generated pyjl_keytype(::Type{T}) where {T} = try; keytype(T); catch; nothing; end; +pyjl_hasvarindices(::Type) = true +pyjl_hasvarindices(::Type{<:AbstractDict}) = false + +pyjl_getindices(x, ko) = + if (K = pyjl_keytype(typeof(x))) !== nothing + ism1(PyObject_Convert(ko, K)) ? PYERR() : (takeresult(K),) + elseif pyjl_hasvarindices(typeof(x)) && PyTuple_Check(ko) + ism1(PyObject_TryConvert(ko, Tuple)) ? PYERR() : takeresult(Tuple) + else + ism1(PyObject_TryConvert(ko, Any)) ? PYERR() : takeresult(Any) + end + +pyjlany_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + k = pyjl_getindices(x, ko) + k === PYERR() && return PyPtr() + try + PyObject_From(x[k...]) + catch err + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa KeyError && (err.key === k || (err.key,) === k) + PyErr_SetStringFromJuliaError(PyExc_KeyError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +@generated pyjl_valtype(::Type{T}) where {T} = try; valtype(T); catch; try eltype(T); catch; nothing; end; end; + +pyjl_getvalue(x, vo) = + if (V = pyjl_valtype(typeof(x))) !== nothing + ism1(PyObject_Convert(vo, V)) ? PYERR() : takeresult(V) + else + ism1(PyObject_Convert(vo, Any)) ? PYERR() : takeresult(Any) + end + +pyjlany_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + k = pyjl_getindices(x, ko) + k === PYERR() && return Cint(-1) + try + if isnull(vo) + delete!(x, k...) + Cint(0) + else + v = pyjl_getvalue(x, vo) + v === PYERR() && return Cint(-1) + x[k...] = v + Cint(0) + end + catch + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa KeyError && (err.key === k || (err.key,) === k) + PyErr_SetStringFromJuliaError(PyExc_KeyError(), err) + elseif err isa MethodError && err.f === delete! + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlany_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator(PyJuliaValue_GetValue(xo))) + +pyjlany_contains(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return Cint(-1) + r == 0 && return Cint(0) + v = takeresult(eltype(x)) + try + Cint(v in x) + catch err + if err isa MethodError && err.f === :in + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlany_richcompare(xo::PyPtr, yo::PyPtr, op::Cint) = begin + x = PyJuliaValue_GetValue(xo) + r = PyObject_TryConvert(yo, Any) + r == -1 && return PyPtr() + r == 0 && return PyNotImplemented_New() + y = takeresult() + try + if op == Py_EQ + PyObject_From(x == y) + elseif op == Py_NE + PyObject_From(x != y) + elseif op == Py_LE + PyObject_From(x <= y) + elseif op == Py_LT + PyObject_From(x < y) + elseif op == Py_GE + PyObject_From(x >= y) + elseif op == Py_GT + PyObject_From(x > y) + else + PyErr_SetString(PyExc_ValueError(), "bad op given to richcompare: $op") + PyPtr() + end + catch err + if err isa MethodError && err.f in (==, !=, <=, <, >=, >) + PyNotImplemented_New() + else + PyErr_SetJuliaError(err) + PyPtr() + end + end +end diff --git a/src/cpython/juliaarray.jl b/src/cpython/juliaarray.jl new file mode 100644 index 00000000..b39ffcf7 --- /dev/null +++ b/src/cpython/juliaarray.jl @@ -0,0 +1,184 @@ +const PyJuliaArrayValue_Type__ref = Ref(PyPtr()) +PyJuliaArrayValue_Type() = begin + ptr = PyJuliaArrayValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.ArrayValue", + base = base, + as_mapping = ( + subscript = pyjlarray_getitem, + ass_subscript = pyjlarray_setitem, + ), + getset = [ + (name="ndim", get=pyjlarray_ndim), + (name="shape", get=pyjlarray_shape), + ], + methods = [ + (name="copy", flags=Py_METH_NOARGS, meth=pyjlarray_copy), + (name="reshape", flags=Py_METH_O, meth=pyjlarray_reshape), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyCollectionABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaArrayValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaArrayValue_New(x::AbstractArray) = PyJuliaValue_New(PyJuliaArrayValue_Type(), x) +PyJuliaValue_From(x::AbstractArray) = PyJuliaArrayValue_New(x) + +pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, ko::PyPtr) = begin + if PySlice_Check(ko) + ao, co, bo = PySimpleObject_GetValue(ko, Tuple{PyPtr, PyPtr, PyPtr}) + # start + r = PyObject_TryConvert(ao, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + a = takeresult(Union{Int, Nothing}) + # step + r = PyObject_TryConvert(bo, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + b = takeresult(Union{Int, Nothing}) + # stop + r = PyObject_TryConvert(co, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + c = takeresult(Union{Int, Nothing}) + # step defaults to 1 + b′ = b === nothing ? 1 : b + if a === nothing && c === nothing + # when neither is specified, start and stop default to the full range, + # which is reversed when the step is negative + if b′ > 0 + a′ = Int(first(x)) + c′ = Int(last(x)) + elseif b′ < 0 + a′ = Int(last(x)) + c′ = Int(first(x)) + else + PyErr_SetString(PyExc_ValueError(), "step must be non-zero") + return PYERR() + end + else + # start defaults + a′ = Int(a === nothing ? first(x) : a < 0 ? (last(x) + a + 1) : (first(x) + a)) + c′ = Int(c === nothing ? last(x) : c < 0 ? (last(x) + 1 + c - sign(b′)) : (first(x) + c - sign(b′))) + end + r = a′ : b′ : c′ + if !checkbounds(Bool, x, r) + PyErr_SetString(PyExc_IndexError(), "array index out of bounds") + return PYERR() + end + r + else + r = PyObject_TryConvert(ko, Int) + ism1(r) && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "index must be slice or integer, got $(PyType_Name(Py_Type(ko)))"); return PYERR()) + k = takeresult(Int) + k′ = k < 0 ? (last(x) + k + 1) : (first(x) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PYERR()) + k′ + end +end + +pyjl_getarrayindices(x::AbstractArray, ko::PyPtr) = begin + kos = PyTuple_Check(ko) ? [PyTuple_GetItem(ko, i-1) for i in 1:PyTuple_Size(ko)] : [ko] + length(kos) == ndims(x) || (PyErr_SetString(PyExc_TypeError(), "expecting exactly $(ndims(x)) indices, got $(length(kos))"); return PYERR()) + ks = [] + for (i,ko) in enumerate(kos) + k = pyjl_getaxisindex(axes(x, i), ko) + k === PYERR() && return PYERR() + push!(ks, k) + end + ks +end + +pyjlarray_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + k = pyjl_getarrayindices(x, ko) + k === PYERR() && return PyPtr() + try + if all(x -> x isa Int, k) + PyObject_From(x[k...]) + else + PyObject_From(view(x, k...)) + end + catch err + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +pyjlarray_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + k = pyjl_getarrayindices(x, ko) + k === PYERR() && return Cint(-1) + try + if isnull(vo) + deleteat!(x, k...) + Cint(0) + else + ism1(PyObject_Convert(vo, eltype(x))) && return PyPtr() + v = takeresult(eltype(x)) + if all(x -> x isa Int, k) + x[k...] = v + Cint(0) + else + PyErr_SetString(PyExc_TypeError(), "multiple assignment not supported") + Cint(-1) + end + end + catch + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa MethodError && err.f === deleteat! + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlarray_ndim(xo::PyPtr, ::Ptr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(ndims(x)) +end + +pyjlarray_shape(xo::PyPtr, ::Ptr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(size(x)) +end + +pyjlarray_copy(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(copy(x)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlarray_reshape(xo::PyPtr, arg::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractArray + r = PyObject_TryConvert(arg, Union{Int, Tuple{Vararg{Int}}}) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "shape must be an integer or tuple of integers"); return PyPtr()) + PyObject_From(reshape(x, takeresult()...)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end diff --git a/src/cpython/juliadict.jl b/src/cpython/juliadict.jl new file mode 100644 index 00000000..c8ba2296 --- /dev/null +++ b/src/cpython/juliadict.jl @@ -0,0 +1,85 @@ +const PyJuliaDictValue_Type__ref = Ref(PyPtr()) +PyJuliaDictValue_Type() = begin + ptr = PyJuliaDictValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.DictValue", + base = base, + iter = pyjldict_iter, + methods = [ + (name="keys", flags=Py_METH_NOARGS, meth=pyjldict_keys), + (name="values", flags=Py_METH_NOARGS, meth=pyjldict_values), + (name="items", flags=Py_METH_NOARGS, meth=pyjldict_items), + (name="get", flags=Py_METH_VARARGS, meth=pyjldict_get), + ], + as_sequence = ( + contains = pyjldict_contains, + ), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyMutableMappingABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaDictValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaDictValue_New(x::AbstractDict) = PyJuliaValue_New(PyJuliaDictValue_Type(), x) +PyJuliaValue_From(x::AbstractDict) = PyJuliaDictValue_New(x) + +pyjldict_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator(keys(PyJuliaValue_GetValue(xo)::AbstractDict))) + +pyjldict_keys(xo::PyPtr, ::PyPtr) = PyObject_From(keys(PyJuliaValue_GetValue(xo))) +pyjldict_values(xo::PyPtr, ::PyPtr) = PyObject_From(values(PyJuliaValue_GetValue(xo))) +pyjldict_items(xo::PyPtr, ::PyPtr) = PyObject_From(DictPairSet(pairs(PyJuliaValue_GetValue(xo)))) + +pyjldict_contains(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractDict + r = PyObject_TryConvert(vo, keytype(x)) + r == -1 && return Cint(-1) + r == 0 && return Cint(0) + v = takeresult(keytype(x)) + try + Cint(haskey(x, v)) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +pyjldict_get(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractDict + ism1(PyArg_CheckNumArgsBetween("get", args, 1, 2)) && return PyPtr() + ko = PyTuple_GetItem(args, 0) + vo = PyTuple_Size(args) < 2 ? Py_None() : PyTuple_GetItem(args, 1) + r = PyObject_TryConvert(ko, keytype(x)) + r == -1 && return PyPtr() + r == 0 && (Py_IncRef(vo); return vo) + k = takeresult(keytype(x)) + try + if haskey(x, k) + PyObject_From(x[k]) + else + Py_IncRef(vo) + vo + end + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +struct DictPairSet{K,V,T<:AbstractDict{K,V}} <: AbstractSet{Tuple{K,V}} + dict :: T +end +Base.length(x::DictPairSet) = length(x.dict) +Base.iterate(x::DictPairSet) = (r=iterate(x.dict); r===nothing ? nothing : (Tuple(r[1]), r[2])) +Base.iterate(x::DictPairSet, st) = (r=iterate(x.dict, st); r===nothing ? nothing : (Tuple(r[1]), r[2])) +Base.in(v, x::DictPairSet) = v in x.dict diff --git a/src/cpython/juliaiterator.jl b/src/cpython/juliaiterator.jl new file mode 100644 index 00000000..7a971921 --- /dev/null +++ b/src/cpython/juliaiterator.jl @@ -0,0 +1,57 @@ +mutable struct Iterator + val :: Any + st :: Any +end +Iterator(x) = Iterator(x, nothing) +Base.length(x::Iterator) = length(x.val) + +const PyJuliaIteratorValue_Type__ref = Ref(PyPtr()) +PyJuliaIteratorValue_Type() = begin + ptr = PyJuliaIteratorValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.IteratorValue", + base = base, + iter = pyjliter_iter, + iternext = pyjliter_iternext, + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaIteratorValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaIteratorValue_New(x::Iterator) = PyJuliaValue_New(PyJuliaIteratorValue_Type(), x) + +pyjliter_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator((PyJuliaValue_GetValue(xo)::Iterator).val)) + +pyjliter_iternext(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::Iterator + val = x.val + st = x.st + if st === nothing + z = iterate(val) + else + z = iterate(val, something(st)) + end + if z === nothing + PyPtr() + else + r, newst = z + x.st = Some(newst) + PyObject_From(r) + end +catch err + if err isa MethodError && err.f === iterate + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() +end diff --git a/src/cpython/juliatype.jl b/src/cpython/juliatype.jl new file mode 100644 index 00000000..8ca8ffde --- /dev/null +++ b/src/cpython/juliatype.jl @@ -0,0 +1,56 @@ +const PyJuliaTypeValue_Type__ref = Ref(PyPtr()) +PyJuliaTypeValue_Type() = begin + ptr = PyJuliaTypeValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.TypeValue", + base = base, + as_mapping = ( + subscript = pyjltype_getitem, + ass_subscript = pyjltype_setitem, + ), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaTypeValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaTypeValue_New(x::Type) = PyJuliaValue_New(PyJuliaTypeValue_Type(), x) +PyJuliaValue_From(x::Type) = PyJuliaTypeValue_New(x) + +pyjltype_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::Type + if PyTuple_Check(ko) + r = PyObject_Convert(ko, Tuple) + ism1(r) && return PyPtr() + k = takeresult(Tuple) + try + PyObject_From(x{k...}) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + else + r = PyObject_Convert(ko, Any) + ism1(r) && return PyPtr() + k = takeresult(Any) + try + PyObject_From(x{k}) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + end +end + +pyjltype_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + PyErr_SetString(PyExc_TypeError(), "Not supported.") + PyErr() +end diff --git a/src/cpython/juliavector.jl b/src/cpython/juliavector.jl new file mode 100644 index 00000000..86430c0b --- /dev/null +++ b/src/cpython/juliavector.jl @@ -0,0 +1,50 @@ +const PyJuliaVectorValue_Type__ref = Ref(PyPtr()) +PyJuliaVectorValue_Type() = begin + ptr = PyJuliaVectorValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaArrayValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.VectorValue", + base = base, + methods = [ + (name="resize", flags=Py_METH_O, meth=pyjlvector_resize), + (name="sort", flags=Py_METH_NOARGS, meth=pyjlvector_sort), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyMutableSequenceABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaVectorValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaVectorValue_New(x::AbstractVector) = PyJuliaValue_New(PyJuliaVectorValue_Type(), x) +PyJuliaValue_From(x::AbstractVector) = PyJuliaVectorValue_New(x) + +pyjlvector_resize(xo::PyPtr, arg::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(arg, Int) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "size must be an integer"); return PyPtr()) + resize!(x, takeresult(Int)) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_sort(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + sort!(x) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end From 896864d9f85bc15a9965a86ad3ae6c95e0cf0594 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 23 Dec 2020 12:26:21 +0000 Subject: [PATCH 13/14] julia arrays now support buffer and array interface --- src/PyObject.jl | 32 +++-- src/{old => }/PyObjectArray.jl | 27 ++++- src/Python.jl | 1 + src/cpython/arg.jl | 41 ++++--- src/cpython/juliaany.jl | 4 +- src/cpython/juliaarray.jl | 209 ++++++++++++++++++++++++++++++++- src/cpython/juliavector.jl | 189 ++++++++++++++++++++++++++++- src/utils.jl | 39 +++++- 8 files changed, 504 insertions(+), 38 deletions(-) rename src/{old => }/PyObjectArray.jl (67%) diff --git a/src/PyObject.jl b/src/PyObject.jl index f5dc3a4b..25216c2e 100644 --- a/src/PyObject.jl +++ b/src/PyObject.jl @@ -62,6 +62,7 @@ Base.show(io::IO, o::PyObject) = begin end function Base.show(io::IO, ::MIME"text/plain", o::PyObject) + prefix = get(io, :typeinfo, Any) != typeof(o) h, w = displaysize(io) h -= 3 x = try @@ -73,26 +74,29 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject) if '\n' ∈ x # multiple lines # each one is truncated to one screen width - print(io, "py:") - h -= 1 + if prefix + print(io, "py:") + h -= 1 + end xs = split(x, '\n') - printlines(xs) = - for x in xs + printlines(xs, nl=true) = + for (i,x) in enumerate(xs) + (nl || i>1) && print(io, '\n') if length(x) ≤ w - print(io, '\n', x) + print(io, x) else - print(io, '\n', x[1:nextind(x, 0, w-1)], '…') + print(io, x[1:nextind(x, 0, w-1)], '…') end end if length(xs) ≤ h # all lines fit on screen - printlines(xs) + printlines(xs, prefix) else # too many lines, skip the middle ones h -= 1 h2 = cld(h, 2) h3 = (length(xs)+1)-(h-h2) - printlines(xs[1:h2]) + printlines(xs[1:h2], prefix) linelen = min( checkbounds(Bool, xs, h2) ? length(xs[h2]) : 0, checkbounds(Bool, xs, h3) ? length(xs[h3]) : 0, @@ -103,14 +107,16 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject) print(io, "\n", pad > 0 ? " "^pad : "", msg) printlines(xs[h3:end]) end - elseif length(x) ≤ w-4 + elseif length(x) ≤ (prefix ? w-4 : w) # one short line - print(io, "py: ", x) + print(io, prefix ? "py: " : "", x) return else # one long line - println(io, "py:") - h -= 1 + if prefix + println(io, "py:") + h -= 1 + end a = h * w if length(x) ≤ a # whole string fits on screen @@ -127,7 +133,7 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject) end end else - print(io, "py: ", x) + print(io, prefix ? "py: " : "", x) end end diff --git a/src/old/PyObjectArray.jl b/src/PyObjectArray.jl similarity index 67% rename from src/old/PyObjectArray.jl rename to src/PyObjectArray.jl index 9a5f62d7..d6ad90fd 100644 --- a/src/old/PyObjectArray.jl +++ b/src/PyObjectArray.jl @@ -1,3 +1,11 @@ +""" + PyObjectArray(undef, dims...) + PyObjectArray(array) + +An array of `PyObject`s which supports the Python buffer protocol. + +Internally, the objects are stored as an array of pointers. +""" mutable struct PyObjectArray{N} <: AbstractArray{PyObject, N} ptrs :: Array{CPyPtr, N} function PyObjectArray{N}(::UndefInitializer, dims::NTuple{N,Integer}) where {N} @@ -34,10 +42,11 @@ function Base.getindex(x::PyObjectArray, i::Integer...) end function Base.setindex!(x::PyObjectArray, _v, i::Integer...) - v = convert(PyObject, _v) + vo = C.PyObject_From(_v) + isnull(vo) && pythrow() C.Py_DecRef(x.ptrs[i...]) - pyincref!(v) - x.ptrs[i...] = pyptr(v) + C.Py_IncRef(vo) + x.ptrs[i...] = vo x end @@ -46,3 +55,15 @@ function Base.deleteat!(x::PyObjectArray, i::Integer) deleteat!(x.ptrs, i) x end + +C._pyjlarray_get_buffer(o, buf, flags, x::PyObjectArray) = + C.pyjl_get_buffer_impl(o, buf, flags, pointer(x.ptrs), sizeof(CPyPtr), length(x), ndims(x), "O", size(x), strides(x.ptrs), true) + +C._pyjlarray_array_interface(x::PyObjectArray) = + C.PyDict_From(Dict( + "shape" => size(x), + "typestr" => "|O", + "data" => (UInt(pointer(x.ptrs)), false), + "strides" => strides(x.ptrs) .* sizeof(CPyPtr), + "version" => 3, + )) diff --git a/src/Python.jl b/src/Python.jl index 8405f6e2..2c5be1f0 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -94,6 +94,7 @@ include("PySet.jl") include("PyIterable.jl") include("PyIO.jl") include("PyBuffer.jl") +include("PyObjectArray.jl") include("PyPandasDataFrame.jl") include("julia.jl") diff --git a/src/cpython/arg.jl b/src/cpython/arg.jl index 58041f4e..9401698e 100644 --- a/src/cpython/arg.jl +++ b/src/cpython/arg.jl @@ -68,7 +68,7 @@ end struct NODEFAULT end PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}) = - if i !== nothing && !isnull(args) && 0 ≤ i ≤ PyTuple_Size(args) + if i !== nothing && !isnull(args) && 0 ≤ i < PyTuple_Size(args) return PyTuple_GetItem(args, i) elseif k !== nothing && !isnull(kwargs) && (ro = PyDict_GetItemString(kwargs, k)) != PyPtr() return ro @@ -76,34 +76,43 @@ PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,No return PyPtr() end -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Union{Int,Nothing}=nothing, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} = begin +""" + PyArg_GetArg(T, funcname, [args, i], [kwargs, k], [default]) + +Attempt to find and convert the specified argument to a `T`. Return 0 on success, in which case the result can be retrieved with `takeresult(T)`. Return -1 on failure, with a Python error set. + +- `funcname::String` is the name of the function this is used in, for constructing error messages. +- `args::PyPtr` is a tuple of arguments, and `i::Int` an index. +- `kwargs::PyPtr` is a dict of keyword arguments, and `k::String` a key. +- `default` specifies a default value if the argument is not found. NOTE: It need not be a `T`, so when retrieving the result use `takeresult(Union{T,typeof(default)})`. +""" +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, kwargs::PyPtr, k::Union{String,Nothing}, d=NODEFAULT()) where {T} = begin ro = PyArg_Find(args, kwargs, i, k) if isnull(ro) - if k !== nothing + if d !== NODEFAULT() + putresult(d) + return 0 + elseif k !== nothing PyErr_SetString(PyExc_TypeError(), "$name() did not get required argument '$k'") - elseif i !== nothing + return -1 + elseif i !== nothing && i ≥ 0 PyErr_SetString(PyExc_TypeError(), "$name() takes at least $(i+1) arguments (got $(isnull(args) ? 0 : PyTuple_Size(args)))") + return -1 else error("impossible to satisfy this argument") end - return -1 end r = PyObject_TryConvert(ro, T) if r == -1 return -1 elseif r == 0 - if d === NODEFAULT() - PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'") - return -1 - else - putresult(d) - return 0 - end + PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'") + return -1 else return 0 end end -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} = - PyArg_GetArg(T, name, args, PyPtr(), i, k, d) -PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr, k::Union{String,Nothing}, d::Union{T, NODEFAULT}=NODEFAULT()) where {T} = - PyArg_GetArg(T, name, args, kwargs, nothing, k, d) +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Int, d=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, args, i, PyPtr(), nothing, d) +PyArg_GetArg(::Type{T}, name::String, kwargs::PyPtr, k::String, d=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, PyPtr(), nothing, kwargs, k, d) diff --git a/src/cpython/juliaany.jl b/src/cpython/juliaany.jl index c51451ee..795eea32 100644 --- a/src/cpython/juliaany.jl +++ b/src/cpython/juliaany.jl @@ -41,7 +41,9 @@ PyJuliaValue_From(x) = PyJuliaAnyValue_New(x) pyjlany_repr(xo::PyPtr) = try x = PyJuliaValue_GetValue(xo) - s = "" + # s = "" + s = sprint((io,x)->show(io,MIME"text/plain"(),x), x, context=:limit=>true) + s = string("jl:", '\n' in s ? '\n' : ' ', s) PyUnicode_From(s) catch err PyErr_SetJuliaError(err) diff --git a/src/cpython/juliaarray.jl b/src/cpython/juliaarray.jl index b39ffcf7..a579fb88 100644 --- a/src/cpython/juliaarray.jl +++ b/src/cpython/juliaarray.jl @@ -12,13 +12,19 @@ PyJuliaArrayValue_Type() = begin subscript = pyjlarray_getitem, ass_subscript = pyjlarray_setitem, ), + as_buffer = ( + get = pyjlarray_get_buffer, + release = pyjlarray_release_buffer, + ), getset = [ (name="ndim", get=pyjlarray_ndim), (name="shape", get=pyjlarray_shape), + (name="__array_interface__", get=pyjlarray_array_interface), ], methods = [ (name="copy", flags=Py_METH_NOARGS, meth=pyjlarray_copy), (name="reshape", flags=Py_METH_O, meth=pyjlarray_reshape), + (name="__array__", flags=Py_METH_NOARGS, meth=pyjlarray_array), ], )) ptr = PyPtr(pointer(t)) @@ -132,7 +138,7 @@ pyjlarray_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin deleteat!(x, k...) Cint(0) else - ism1(PyObject_Convert(vo, eltype(x))) && return PyPtr() + ism1(PyObject_Convert(vo, eltype(x))) && return Cint(-1) v = takeresult(eltype(x)) if all(x -> x isa Int, k) x[k...] = v @@ -182,3 +188,204 @@ catch err PyErr_SetJuliaError(err) PyPtr() end + +### Buffer Protocol + +isflagset(flags, mask) = (flags & mask) == mask + +const PYJLBUFCACHE = Dict{Ptr{Cvoid}, Any}() + +pyjl_get_buffer_impl(o::PyPtr, buf::Ptr{Py_buffer}, flags, ptr, elsz, len, ndim, fmt, sz, strds, mutable) = begin + b = UnsafePtr(buf) + c = [] + + # not influenced by flags: obj, buf, len, itemsize, ndim + b.obj[] = C_NULL + b.buf[] = ptr + b.itemsize[] = elsz + b.len[] = elsz * len + b.ndim[] = ndim + + # readonly + if isflagset(flags, PyBUF_WRITABLE) + if mutable + b.readonly[] = 1 + else + PyErr_SetString(PyExc_BufferError(), "not writable") + return Cint(-1) + end + else + b.readonly[] = mutable ? 0 : 1 + end + + # format + if isflagset(flags, PyBUF_FORMAT) + b.format[] = cacheptr!(c, fmt) + else + b.format[] = C_NULL + end + + # shape + if isflagset(flags, PyBUF_ND) + b.shape[] = cacheptr!(c, Py_ssize_t[sz...]) + else + b.shape[] = C_NULL + end + + # strides + if isflagset(flags, PyBUF_STRIDES) + b.strides[] = cacheptr!(c, Py_ssize_t[(strds .* elsz)...]) + else + if Python.size_to_cstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not C contiguous and strides not requested") + return Cint(-1) + end + b.strides[] = C_NULL + end + + # check contiguity + if isflagset(flags, PyBUF_C_CONTIGUOUS) + if Python.size_to_cstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not C contiguous") + return Cint(-1) + end + end + if isflagset(flags, PyBUF_F_CONTIGUOUS) + if Python.size_to_fstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not Fortran contiguous") + return Cint(-1) + end + end + if isflagset(flags, PyBUF_ANY_CONTIGUOUS) + if Python.size_to_cstrides(1, sz...) != strds && Python.size_to_fstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not contiguous") + return Cint(-1) + end + end + + # suboffsets + b.suboffsets[] = C_NULL + + # internal + cptr = Base.pointer_from_objref(c) + PYJLBUFCACHE[cptr] = c + b.internal[] = cptr + + # obj + Py_IncRef(o) + b.obj[] = o + Cint(0) +end + +pyjlarray_isbufferabletype(::Type{T}) where {T} = + T in (Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}, Bool, Ptr{Cvoid}) +pyjlarray_isbufferabletype(::Type{T}) where {T<:Tuple} = + isconcretetype(T) && Base.allocatedinline(T) && all(pyjlarray_isbufferabletype, fieldtypes(T)) +pyjlarray_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = + pyjlarray_isbufferabletype(T) + +_pyjlarray_get_buffer(xo, buf, flags, x::AbstractArray) = try + if pyjlarray_isbufferabletype(eltype(x)) + pyjl_get_buffer_impl(xo, buf, flags, Base.unsafe_convert(Ptr{eltype(x)}, x), Base.aligned_sizeof(eltype(x)), length(x), ndims(x), Python.pybufferformat(eltype(x)), size(x), strides(x), Python.ismutablearray(x)) + else + error("element type is not bufferable") + end +catch err + PyErr_SetString(PyExc_BufferError(), "Buffer protocol not supported by Julia '$(typeof(x))' (details: $err)") + Cint(-1) +end + +pyjlarray_get_buffer(xo::PyPtr, buf::Ptr{Py_buffer}, flags::Cint) = + _pyjlarray_get_buffer(xo, buf, flags, PyJuliaValue_GetValue(xo)::AbstractArray) + +pyjlarray_release_buffer(xo::PyPtr, buf::Ptr{Py_buffer}) = begin + delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) + nothing +end + +### Array Interface + +pyjlarray_isarrayabletype(::Type{T}) where {T} = + T in (UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64, Bool, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}) +pyjlarray_isarrayabletype(::Type{T}) where {T<:Tuple} = + isconcretetype(T) && Base.allocatedinline(T) && all(pyjlarray_isarrayabletype, T.parameters) +pyjlarray_isarrayabletype(::Type{NamedTuple{names,types}}) where {names,types} = + pyjlarray_isarrayabletype(types) + +pyjlarray_array_interface(xo::PyPtr, ::Ptr{Cvoid}) = + _pyjlarray_array_interface(PyJuliaValue_GetValue(xo)::AbstractArray) + +PyDescrObject_From(fields) = begin + ro = PyList_New(0) + for (name, descr) in fields + descro = descr isa String ? PyUnicode_From(descr) : PyDescrObject_From(descr) + isnull(descro) && (Py_DecRef(ro); return PyPtr()) + fieldo = PyTuple_From((name, PyObjectRef(descro))) + Py_DecRef(descro) + isnull(fieldo) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, fieldo) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + ro +end + +_pyjlarray_array_interface(x::AbstractArray) = try + if pyjlarray_isarrayabletype(eltype(x)) + # gather information + shape = size(x) + data = (UInt(Base.unsafe_convert(Ptr{eltype(x)}, x)), !Python.ismutablearray(x)) + strides = Base.strides(x) .* Base.aligned_sizeof(eltype(x)) + version = 3 + typestr, descr = Python.pytypestrdescr(eltype(x)) + isempty(typestr) && error("invalid element type") + # make the dictionary + d = PyDict_From(Dict( + "shape" => shape, + "typestr" => typestr, + "data" => data, + "strides" => strides, + "version" => version, + )) + isnull(d) && return PyPtr() + if descr !== nothing + descro = PyDescrObject_From(descr) + err = PyDict_SetItemString(d, "descr", descro) + Py_DecRef(descro) + ism1(err) && (Py_DecRef(d); return PyPtr()) + end + d + else + error("invalid element type") + end +catch err + PyErr_SetString(PyExc_AttributeError(), "__array_interface__") + PyPtr() +end + +pyjlarray_array(xo::PyPtr, ::PyPtr) = begin + if PyObject_HasAttrString(xo, "__array_interface__") == 0 + # convert to a PyObjectArray + x = PyJuliaValue_GetValue(xo)::AbstractArray + y = try + Python.PyObjectArray(x) + catch err + PyErr_SetJuliaError(err) + return PyPtr() + end + yo = PyJuliaArrayValue_New(y) + isnull(yo) && return PyPtr() + else + # already supports the array interface + Py_IncRef(xo) + yo = xo + end + npo = PyImport_ImportModule("numpy") + isnull(npo) && (Py_DecRef(yo); return PyPtr()) + aao = PyObject_GetAttrString(npo, "array") + Py_DecRef(npo) + isnull(aao) && (Py_DecRef(yo); return PyPtr()) + ao = PyObject_CallNice(aao, PyObjectRef(yo)) + Py_DecRef(aao) + Py_DecRef(yo) + ao +end diff --git a/src/cpython/juliavector.jl b/src/cpython/juliavector.jl index 86430c0b..fb246290 100644 --- a/src/cpython/juliavector.jl +++ b/src/cpython/juliavector.jl @@ -10,7 +10,17 @@ PyJuliaVectorValue_Type() = begin base = base, methods = [ (name="resize", flags=Py_METH_O, meth=pyjlvector_resize), - (name="sort", flags=Py_METH_NOARGS, meth=pyjlvector_sort), + (name="sort", flags=Py_METH_KEYWORDS|Py_METH_VARARGS, meth=pyjlvector_sort), + (name="reverse", flags=Py_METH_NOARGS, meth=pyjlvector_reverse), + (name="clear", flags=Py_METH_NOARGS, meth=pyjlvector_clear), + (name="__reversed__", flags=Py_METH_NOARGS, meth=pyjlvector_reversed), + (name="insert", flags=Py_METH_VARARGS, meth=pyjlvector_insert), + (name="append", flags=Py_METH_O, meth=pyjlvector_append), + (name="extend", flags=Py_METH_O, meth=pyjlvector_extend), + (name="pop", flags=Py_METH_VARARGS, meth=pyjlvector_pop), + (name="remove", flags=Py_METH_O, meth=pyjlvector_remove), + (name="index", flags=Py_METH_O, meth=pyjlvector_index), + (name="count", flags=Py_METH_O, meth=pyjlvector_count), ], )) ptr = PyPtr(pointer(t)) @@ -40,11 +50,184 @@ catch err PyPtr() end -pyjlvector_sort(xo::PyPtr, ::PyPtr) = try +pyjlvector_sort(xo::PyPtr, args::PyPtr, kwargs::PyPtr) = try x = PyJuliaValue_GetValue(xo)::AbstractVector - sort!(x) + ism1(PyArg_CheckNumArgsEq("sort", args, 0)) && return PyPtr() + ism1(PyArg_GetArg(Bool, "sort", kwargs, "reverse", false)) && return PyPtr() + rev = takeresult(Bool) + ism1(PyArg_GetArg(Any, "sort", kwargs, "key", nothing)) && return PyPtr() + by = takeresult() + sort!(x, rev=rev, by= by===nothing ? identity : by) PyNone_New() catch err PyErr_SetJuliaError(err) PyPtr() end + +pyjlvector_reverse(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + reverse!(x) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_clear(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + empty!(x) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_reversed(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + PyObject_From(reverse(x)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_insert(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + a = axes(x, 1) + ism1(PyArg_CheckNumArgsEq("insert", args, 2)) && return PyPtr() + ism1(PyArg_GetArg(Int, "insert", args, 0)) && return PyPtr() + k = takeresult(Int) + k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PyPtr()) + ism1(PyArg_GetArg(eltype(x), "insert", args, 1)) && return PyPtr() + v = takeresult(eltype(x)) + try + insert!(x, k′, v) + PyNone_New() + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_append(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "array value must be a Julia '$(eltype(x))', not a '$(PyType_Name(Py_Type(vo)))'"); return PyPtr()) + v = takeresult(eltype(x)) + try + push!(x, v) + PyNone_New() + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_extend(xo::PyPtr, vso::PyPtr) = begin + # TODO: this might be quicker if we introduce a function barrier so typeof(x) is known + x = PyJuliaValue_GetValue(xo)::AbstractVector + it = PyObject_GetIter(vso) + isnull(it) && return PyPtr() + try + while true + vo = PyIter_Next(it) + if !isnull(vo) + r = PyObject_TryConvert(vo, eltype(x)) + Py_DecRef(vo) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "array value must be a Julia '$(eltype(x))', not a '$(PyType_Name(Py_Type(vo)))'"); return PyPtr()) + push!(x, takeresult(eltype(x))) + elseif PyErr_IsSet() + return PyPtr() + else + return PyNone_New() + end + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + finally + Py_DecRef(it) + end +end + +pyjlvector_pop(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + a = axes(x, 1) + ism1(PyArg_CheckNumArgsLe("pop", args, 1)) && return PyPtr() + ism1(PyArg_GetArg(Int, "pop", args, 0, -1)) && return PyPtr() + k = takeresult(Int) + k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PyPtr()) + try + if k′ == last(a) + v = pop!(x) + elseif k′ == first(a) + v = popfirst!(x) + else + v = x[k′] + deleteat!(x, k′) + end + PyObject_From(v) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_remove(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_ValueError(), "value not in vector"); return PyPtr()) + v = takeresult(eltype(x)) + try + k = findfirst(x->x==v, x) + if k === nothing + PyErr_SetString(PyExc_ValueError(), "value not in vector") + PyPtr() + else + deleteat!(x, k) + PyNone_New() + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_index(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_ValueError(), "value not in vector"); return PyPtr()) + v = takeresult(eltype(x)) + try + k = findfirst(x->x==v, x) + if k === nothing + PyErr_SetString(PyExc_ValueError(), "value not in vector") + PyPtr() + else + PyObject_From(k - first(axes(x, 1))) + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_count(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && return PyObject_From(0) + v = takeresult(eltype(x)) + try + n = count(x->x==v, x) + PyObject_From(n) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end diff --git a/src/utils.jl b/src/utils.jl index 4b4d1dc0..b5cdc1b3 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,6 +28,7 @@ pybufferformat(::Type{T}) where {T} = T == Complex{Float64} ? "=Zd" : T == Bool ? "?" : T == Ptr{Cvoid} ? "P" : + T == C.PyObjectRef ? "O" : if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) n = fieldcount(T) flds = [] @@ -60,12 +61,48 @@ pybufferformat_to_type(fmt::AbstractString) = fmt == "d" ? Cdouble : fmt == "?" ? Bool : fmt == "P" ? Ptr{Cvoid} : - fmt == "O" ? CPyObjRef : + fmt == "O" ? C.PyObjectRef : fmt == "=e" ? Float16 : fmt == "=f" ? Float32 : fmt == "=d" ? Float64 : error("not implemented: $(repr(fmt))") +islittleendian() = Base.ENDIAN_BOM == 0x04030201 ? true : Base.ENDIAN_BOM == 0x01020304 ? false : error() + +pytypestrdescr(::Type{T}) where {T} = begin + c = islittleendian() ? '<' : '>' + T == Bool ? ("$(c)b$(sizeof(Bool))", nothing) : + T == Int8 ? ("$(c)i1", nothing) : + T == UInt8 ? ("$(c)u1", nothing) : + T == Int16 ? ("$(c)i2", nothing) : + T == UInt16 ? ("$(c)u2", nothing) : + T == Int32 ? ("$(c)i4", nothing) : + T == UInt32 ? ("$(c)u4", nothing) : + T == Int64 ? ("$(c)i8", nothing) : + T == UInt64 ? ("$(c)u8", nothing) : + T == Float16 ? ("$(c)f2", nothing) : + T == Float32 ? ("$(c)f4", nothing) : + T == Float64 ? ("$(c)f8", nothing) : + T == C.PyObjectRef ? ("|O", nothing) : + if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) + n = fieldcount(T) + flds = [] + for i in 1:n + nm = fieldname(T, i) + tp = fieldtype(T, i) + ts, ds = pytypestrdescr(tp) + isempty(ts) && return ("", nothing) + push!(flds, (nm isa Integer ? "f$(nm-1)" : string(nm), ds === nothing ? ts : ds)) + d = (i==n ? sizeof(T) : fieldoffset(T, i+1)) - (fieldoffset(T, i) + sizeof(tp)) + @assert d≥0 + d>0 && push!(flds, ("", "|V$(d)")) + end + ("|$(sizeof(T))V", flds) + else + ("", nothing) + end +end + ### TYPE UTILITIES # Used to signal a Python error from functions that return general Julia objects From 8f3c10a97b2f6b07c91af7246d6bcb410c377a65 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Wed, 23 Dec 2020 17:20:23 +0000 Subject: [PATCH 14/14] dev --- src/PyArray.jl | 194 +++++++++++++++++++++++++++++++++++++ src/PyObject.jl | 9 +- src/PyPandasDataFrame.jl | 61 ++++++------ src/Python.jl | 1 + src/builtins.jl | 10 +- src/cpython/CPython.jl | 1 + src/cpython/juliaany.jl | 38 +++++++- src/cpython/juliamodule.jl | 53 ++++++++++ src/cpython/object.jl | 29 +++++- src/init.jl | 64 ++++++------ src/old/PyArray.jl | 138 -------------------------- src/utils.jl | 55 +++++++++++ 12 files changed, 443 insertions(+), 210 deletions(-) create mode 100644 src/PyArray.jl create mode 100644 src/cpython/juliamodule.jl delete mode 100644 src/old/PyArray.jl diff --git a/src/PyArray.jl b/src/PyArray.jl new file mode 100644 index 00000000..8e9b013b --- /dev/null +++ b/src/PyArray.jl @@ -0,0 +1,194 @@ +""" + PyArray{T,N,R,M,L}(o) + +Interpret the Python array `o` as a Julia array. + +The input may be anything supporting the buffer protocol or the numpy array interface. +This includes, `bytes`, `bytearray`, `array.array`, `numpy.ndarray`, `pandas.Series`. + +- `T` is the (Julia) element type. +- `N` is the number of dimensions. +- `R` is the type of elements of the underlying buffer (which may be different from `T` to allow some basic conversion). +- `M` is true if the array is mutable. +- `L` is true if the array supports fast linear indexing. +""" +mutable struct PyArray{T,N,R,M,L} <: AbstractArray{T,N} + ref :: PyRef + ptr :: Ptr{R} + size :: NTuple{N,Int} + length :: Int + bytestrides :: NTuple{N,Int} + handle :: Any +end +const PyVector{T,R,M,L} = PyArray{T,1,R,M,L} +const PyMatrix{T,R,M,L} = PyArray{T,2,R,M,L} +export PyArray, PyVector, PyMatrix + +ispyreftype(::Type{<:PyArray}) = true +pyptr(x::PyArray) = pyptr(x.ref) +Base.unsafe_convert(::Type{CPyPtr}, x::PyArray) = pyptr(x.ref) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyArray} = CTryConvertRule_trywrapref(o, T) + +function PyArray{T,N,R,M,L}(o::PyRef, info) where {T,N,R,M,L} + # T - array element type + T isa Type || error("T must be a type, got T=$T") + + # N - number of dimensions + N isa Integer || error("N must be an integer, got N=$N") + N isa Int || return PyArray{T, Int(N), R, M, L}(o, info) + N == info.ndims || error("source dimension is $(info.ndims), got N=$N") + + # R - buffer element type + R isa Type || error("R must be a type, got R=$R") + Base.allocatedinline(R) || error("source elements must be allocated inline, got R=$R") + Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R") + + # M - mutable + M isa Bool || error("M must be true or false, got M=$M") + !M || info.mutable || error("source is immutable, got M=$M") + + bytestrides = info.bytestrides + size = info.size + + # L - linear indexable + L isa Bool || error("L must be true or false, got L=$L") + !L || N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable, got L=$L") + + PyArray{T, N, R, M, L}(PyRef(o), Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle) +end +PyArray{T,N,R,M}(o::PyRef, info) where {T,N,R,M} = PyArray{T,N,R,M, N≤1 || size_to_fstrides(info.bytestrides[1], info.size...) == info.bytestrides}(o, info) +PyArray{T,N,R}(o::PyRef, info) where {T,N,R} = PyArray{T,N,R, info.mutable}(o, info) +PyArray{T,N}(o::PyRef, info) where {T,N} = PyArray{T,N, info.eltype}(o, info) +PyArray{T}(o::PyRef, info) where {T} = PyArray{T, info.ndims}(o, info) +PyArray{<:Any,N}(o::PyRef, info) where {N} = PyArray{pyarray_default_T(info.eltype), N}(o, info) +PyArray(o::PyRef, info) = PyArray{pyarray_default_T(info.eltype)}(o, info) + +(::Type{A})(o; opts...) where {A<:PyArray} = begin + ref = PyRef(o) + info = pyarray_info(ref; opts...) + info = ( + ndims = Int(info.ndims), + eltype = info.eltype :: Type, + elsize = Int(info.elsize), + mutable = info.mutable :: Bool, + bytestrides = NTuple{Int(info.ndims), Int}(info.bytestrides), + size = NTuple{Int(info.ndims), Int}(info.size), + ptr = Ptr{Cvoid}(info.ptr), + handle = info.handle, + ) + A(ref, info) +end + +function pyarray_info(ref; buffer=true, array=true, copy=true) + if array && pyhasattr(ref, "__array_interface__") + pyconvertdescr(x) = begin + @py ``` + def convert(x): + def fix(x): + a = x[0] + a = (a, a) if isinstance(a, str) else (a[0], a[1]) + b = x[1] + c = x[2] if len(x)>2 else 1 + return (a, b, c) + if x is None or isinstance(x, str): + return x + else: + return [fix(y) for y in x] + $(r::Union{Nothing,String,Vector{Tuple{Tuple{String,String}, PyObject, Int}}}) = convert($x) + ``` + r isa Vector ? [(a, pyconvertdescr(b), c) for (a,b,c) in r] : r + end + ai = pygetattr(ref, "__array_interface__") + pyconvert(Int, ai["version"]) == 3 || error("wrong version") + size = pyconvert(Tuple{Vararg{Int}}, ai["shape"]) + ndims = length(size) + typestr = pyconvert(String, ai["typestr"]) + descr = pyconvertdescr(ai.get("descr")) + eltype = pytypestrdescr_to_type(typestr, descr) + elsize = Base.aligned_sizeof(eltype) + strides = pyconvert(Union{Nothing, Tuple{Vararg{Int}}}, ai.get("strides")) + strides === nothing && (strides = size_to_cstrides(elsize, size...)) + pyis(ai.get("mask"), pynone()) || error("mask not supported") + offset = pyconvert(Union{Nothing, Int}, ai.get("offset")) + offset === nothing && (offset = 0) + data = pyconvert(Union{PyObject, Tuple{UInt, Bool}, Nothing}, ai.get("data")) + if data isa Tuple + ptr = Ptr{Cvoid}(data[1]) + mutable = !data[2] + handle = (ref, ai) + else + buf = PyBuffer(data === nothing ? ref : data) + ptr = buf.buf + mutable = !buf.readonly + handle = (ref, ai, buf) + end + return (ndims=ndims, eltype=eltype, elsize=elsize, mutable=mutable, bytestrides=strides, size=size, ptr=ptr, handle=handle) + end + if array && pyhasattr(ref, "__array_struct__") + # TODO + end + if buffer && C.PyObject_CheckBuffer(ref) + try + b = PyBuffer(ref, C.PyBUF_RECORDS_RO) + return (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b) + catch + end + end + if array && copy && pyhasattr(ref, "__array__") + try + return pyarray_info(pycall(PyRef, pygetattr(PyRef, ref, "__array__")); buffer=buffer, array=array, copy=false) + catch + end + end + error("given object does not support the buffer protocol or array interface") +end + +Base.isimmutable(x::PyArray{T,N,R,M,L}) where {T,N,R,M,L} = !M +Base.size(x::PyArray) = x.size +Base.length(x::PyArray) = x.length +Base.IndexStyle(::Type{PyArray{T,N,R,M,L}}) where {T,N,R,M,L} = L ? Base.IndexLinear() : Base.IndexCartesian() + +Base.@propagate_inbounds Base.getindex(x::PyArray{T,N,R,M,L}, i::Vararg{Int,N2}) where {T,N,R,M,L,N2} = + if (N2==N) || (L && N2==1) + @boundscheck checkbounds(x, i...) + pyarray_load(T, x.ptr + pyarray_offset(x, i...)) + else + invoke(getindex, Tuple{AbstractArray{T,N}, Vararg{Int,N2}}, x, i...) + end + +Base.@propagate_inbounds Base.setindex!(x::PyArray{T,N,R,true,L}, v, i::Vararg{Int,N2}) where {T,N,R,L,N2} = + if (N2==N) || (L && N2==1) + @boundscheck checkbounds(x, i...) + pyarray_store!(x.ptr + pyarray_offset(x, i...), convert(T, v)) + x + else + invoke(setindex!, Tuple{AbstractArray{T,N}, typeof(v), Vararg{Int,N2}}, x, v, i...) + end + +pyarray_default_T(::Type{R}) where {R} = R +pyarray_default_T(::Type{C.PyObjectRef}) = PyObject + +pyarray_load(::Type{T}, p::Ptr{T}) where {T} = unsafe_load(p) +pyarray_load(::Type{T}, p::Ptr{C.PyObjectRef}) where {T} = begin + o = unsafe_load(p).ptr + isnull(o) && throw(UndefRefError()) + ism1(C.PyObject_Convert(o, T)) && pythrow() + takeresult(T) +end + +pyarray_store!(p::Ptr{T}, v::T) where {T} = unsafe_store!(p, v) +pyarray_store!(p::Ptr{C.PyObjectRef}, v::T) where {T} = begin + o = C.PyObject_From(v) + isnull(o) && pythrow() + C.Py_DecRef(unsafe_load(p).ptr) + unsafe_store!(p, C.PyObjectRef(o)) +end + +pyarray_offset(x::PyArray{T,N,R,M,true}, i::Int) where {T,N,R,M} = + N==0 ? 0 : (i-1) * x.bytestrides[1] + +pyarray_offset(x::PyArray{T,1,R,M,true}, i::Int) where {T,R,M} = + (i-1) .* x.bytestrides[1] + +pyarray_offset(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = + sum((i .- 1) .* x.bytestrides) diff --git a/src/PyObject.jl b/src/PyObject.jl index 25216c2e..306a64f0 100644 --- a/src/PyObject.jl +++ b/src/PyObject.jl @@ -29,12 +29,11 @@ pylazyobject(mk) = PyObject(Val(:lazy), mk) C.PyObject_TryConvert__initial(o, ::Type{PyObject}) = C.putresult(pyborrowedobject(o)) -Base.convert(::Type{Any}, x::PyObject) = x Base.convert(::Type{PyObject}, x::PyObject) = x +Base.convert(::Type{Any}, x::PyObject) = x +Base.convert(::Type{T}, x::PyObject) where {T} = x isa T ? x : pyconvert(T, x) Base.convert(::Type{PyObject}, x) = PyObject(x) -Base.convert(::Type{T}, x::PyObject) where {T} = pyconvert(T, x) - ### Cache some common values const _pynone = pylazyobject(() -> pynone(PyRef)) @@ -137,8 +136,8 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject) end end -Base.show(io::IO, mime::MIME, o::PyObject) = _py_mime_show(io, mime, o) -Base.showable(mime::MIME, o::PyObject) = _py_mime_showable(mime, o) +Base.show(io::IO, mime::_py_mimetype, o::PyObject) = _py_mime_show(io, mime, o) +Base.showable(mime::_py_mimetype, o::PyObject) = _py_mime_showable(mime, o) ### PROPERTIES diff --git a/src/PyPandasDataFrame.jl b/src/PyPandasDataFrame.jl index b3ab4db5..590471b3 100644 --- a/src/PyPandasDataFrame.jl +++ b/src/PyPandasDataFrame.jl @@ -1,37 +1,38 @@ -# const pypandas = pylazyobject(() -> pyimport("pandas")) -# const pypandasdataframetype = pylazyobject(() -> pypandas.DataFrame) +asvector(x::AbstractVector) = x +asvector(x) = collect(x) -# asvector(x::AbstractVector) = x -# asvector(x) = collect(x) - -# """ -# pycolumntable(src) +""" + pycolumntable([T=PyObject,] src) :: T -# Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. -# """ -# function pycolumntable(src) -# cols = Tables.columns(src) -# pydict_fromiter(pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) -# end -# pycolumntable(; cols...) = pycolumntable(cols) -# export pycolumntable +Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. +""" +function pycolumntable(::Type{T}, src) where {T} + cols = Tables.columns(src) + pydict(T, pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) +end +pycolumntable(::Type{T}; cols...) where {T} = pycolumntable(T, cols) +pycolumntable(src) = pycolumntable(PyObject, src) +pycolumntable(; opts...) = pycolumntable(PyObject, opts) +export pycolumntable -# """ -# pyrowtable(src) +""" + pyrowtable([T=PyObject,] src) :: T -# Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. -# """ -# function pyrowtable(src) -# rows = Tables.rows(src) -# names = Tables.columnnames(rows) -# pynames = [pystr(String(n)) for n in names] -# pylist_fromiter(pydict_fromiter(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) -# end -# pyrowtable(; cols...) = pyrowtable(cols) -# export pyrowtable +Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. +""" +function pyrowtable(::Type{T}, src) where {T} + rows = Tables.rows(src) + names = Tables.columnnames(rows) + pynames = [pystr(String(n)) for n in names] + pylist(T, pydict(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) +end +pyrowtable(::Type{T}; cols...) where {T} = pyrowtable(T, cols) +pyrowtable(src) = pyrowtable(PyObject, src) +pyrowtable(; opts...) = pyrowtable(PyObject, opts) +export pyrowtable # """ -# pypandasdataframe([src]; ...) +# pypandasdataframe([T=PyObject,] [src]; ...) :: T # Construct a pandas dataframe from `src`. @@ -77,8 +78,8 @@ Base.unsafe_convert(::Type{CPyPtr}, df::PyPandasDataFrame) = checknull(pyptr(df) C.PyObject_TryConvert__initial(o, ::Type{PyPandasDataFrame}) = C.putresult(PyPandasDataFrame(pyborrowedref(o))) Base.show(io::IO, x::PyPandasDataFrame) = print(io, pystr(String, x)) -Base.show(io::IO, mime::MIME, o::PyPandasDataFrame) = _py_mime_show(io, mime, o) -Base.showable(mime::MIME, o::PyPandasDataFrame) = _py_mime_showable(mime, o) +Base.show(io::IO, mime::_py_mimetype, o::PyPandasDataFrame) = _py_mime_show(io, mime, o) +Base.showable(mime::_py_mimetype, o::PyPandasDataFrame) = _py_mime_showable(mime, o) ### Tables.jl / TableTraits.jl integration diff --git a/src/Python.jl b/src/Python.jl index 2c5be1f0..64b22d6b 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -94,6 +94,7 @@ include("PySet.jl") include("PyIterable.jl") include("PyIO.jl") include("PyBuffer.jl") +include("PyArray.jl") include("PyObjectArray.jl") include("PyPandasDataFrame.jl") diff --git a/src/builtins.jl b/src/builtins.jl index 1aefba10..bd77e524 100644 --- a/src/builtins.jl +++ b/src/builtins.jl @@ -779,23 +779,25 @@ export pyellipsis ### MULTIMEDIA DISPLAY -for (mime, method) in [ +const _py_mimes = [ (MIME"text/html", "_repr_html_"), (MIME"text/markdown", "_repr_markdown_"), (MIME"text/json", "_repr_json_"), (MIME"application/javascript", "_repr_javascript_"), (MIME"application/pdf", "_repr_pdf_"), (MIME"image/jpeg", "_repr_jpeg_"), (MIME"image/png", "_repr_png_"), (MIME"image/svg+xml", "_repr_svg_"), (MIME"text/latex", "_repr_latex_") - ] +] +const _py_mimetype = Union{map(first, _py_mimes)...} +for (mime, method) in _py_mimes T = istextmime(mime()) ? String : Vector{UInt8} @eval begin _py_mime_show(io::IO, mime::$mime, o) = begin try - x = pycall(PyRef, pygetattr(Ref, o, $method)) + x = pycall(PyRef, pygetattr(PyRef, o, $method)) pyis(x, pynone(PyRef)) || return write(io, pyconvert($T, x)) catch end - throw(MethodError(show, (io, mime, o))) + throw(MethodError(_py_mime_show, (io, mime, o))) end _py_mime_showable(::$mime, o) = begin try diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl index de890378..06aa3db0 100644 --- a/src/cpython/CPython.jl +++ b/src/cpython/CPython.jl @@ -62,6 +62,7 @@ include("juliatype.jl") include("juliadict.jl") include("juliaarray.jl") include("juliavector.jl") +include("juliamodule.jl") include("arg.jl") __init__() = begin diff --git a/src/cpython/juliaany.jl b/src/cpython/juliaany.jl index 795eea32..a322b82a 100644 --- a/src/cpython/juliaany.jl +++ b/src/cpython/juliaany.jl @@ -25,6 +25,15 @@ PyJuliaAnyValue_Type() = begin ), methods = [ (name="__dir__", flags=Py_METH_NOARGS, meth=pyjlany_dir), + (name="_repr_html_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/html"))), + (name="_repr_markdown_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/markdown"))), + (name="_repr_json_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/json"))), + (name="_repr_javascript_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("application/javascript"))), + (name="_repr_pdf_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("application/pdf"))), + (name="_repr_jpeg_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/jpeg"))), + (name="_repr_png_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/png"))), + (name="_repr_svg_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/svg+xml"))), + (name="_repr_latex_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/latex"))), ], )) ptr = PyPtr(pointer(t)) @@ -42,7 +51,10 @@ PyJuliaValue_From(x) = PyJuliaAnyValue_New(x) pyjlany_repr(xo::PyPtr) = try x = PyJuliaValue_GetValue(xo) # s = "" - s = sprint((io,x)->show(io,MIME"text/plain"(),x), x, context=:limit=>true) + io = IOBuffer() + ioc = IOContext(io, :limit=>true, :compact=>true, :color=>true) + show(ioc, MIME("text/plain"), x) + s = String(take!(io)) s = string("jl:", '\n' in s ? '\n' : ' ', s) PyUnicode_From(s) catch err @@ -316,3 +328,27 @@ pyjlany_richcompare(xo::PyPtr, yo::PyPtr, op::Cint) = begin end end end + +struct pyjlany_repr_mime{M<:MIME} + mime :: M +end +(f::pyjlany_repr_mime{M})(xo::PyPtr, ::PyPtr) where {M} = begin + x = PyJuliaValue_GetValue(xo) + io = IOBuffer() + try + show(io, f.mime, x) + catch err + if err isa MethodError && err.f === show && err.args === (io, f.mime, x) + return PyNone_New() + else + PyErr_SetJuliaError(err) + return PyPtr() + end + end + data = take!(io) + if istextmime(f.mime) + PyUnicode_From(data) + else + PyBytes_From(data) + end +end diff --git a/src/cpython/juliamodule.jl b/src/cpython/juliamodule.jl new file mode 100644 index 00000000..29ed622a --- /dev/null +++ b/src/cpython/juliamodule.jl @@ -0,0 +1,53 @@ +const PyJuliaModuleValue_Type__ref = Ref(PyPtr()) +PyJuliaModuleValue_Type() = begin + ptr = PyJuliaModuleValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.ModuleValue", + base = base, + methods = [ + (name="eval", flags=Py_METH_VARARGS, meth=pyjlmodule_eval), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaModuleValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaModuleValue_New(x::Module) = PyJuliaValue_New(PyJuliaModuleValue_Type(), x) +PyJuliaValue_From(x::Module) = PyJuliaModuleValue_New(x) + +pyjlmodule_eval(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::Module + ism1(PyArg_CheckNumArgsBetween("eval", args, 1, 2)) && return PyPtr() + if PyTuple_Size(args) == 1 + m = PyJuliaValue_GetValue(xo)::Module + exo = PyTuple_GetItem(args, 0) + else + ism1(PyArg_GetArg(Module, "eval", args, 0)) && return PyPtr() + m = takeresult(Module) + exo = PyTuple_GetItem(args, 1) + end + try + if PyUnicode_Check(exo) + c = PyUnicode_AsString(exo) + isempty(c) && PyErr_IsSet() && return PyPtr() + ex = Meta.parse(c) + else + ism1(PyObject_Convert(exo, Any)) && return PyPtr() + ex = takeresult() + end + r = Base.eval(m, ex) + PyObject_From(r) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end diff --git a/src/cpython/object.jl b/src/cpython/object.jl index 48188192..466113d5 100644 --- a/src/cpython/object.jl +++ b/src/cpython/object.jl @@ -186,16 +186,35 @@ PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin # remove empty lists filter!(x -> !isempty(x), basemros) end - allnames = map(PyType_FullName, alltypes) - # as a special case, we check for the buffer type explicitly + # check the original MRO is preserved + @assert filter(x -> x in tmro, alltypes) == tmro + # some special cases + extranames = Dict() + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array_struct__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array_interface__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end for (i,b) in reverse(collect(enumerate(alltypes))) if PyType_CheckBuffer(b) - insert!(allnames, i+1, "") + push!(get!(Vector, extranames, i), "") break end end - # check the original MRO is preserved - @assert filter(x -> x in tmro, alltypes) == tmro + allnames = [n for (i,t) in enumerate(alltypes) for n in [PyType_FullName(t); get(Vector, extranames, i)]] ### STAGE 2: Get a list of applicable conversion rules. # diff --git a/src/init.jl b/src/init.jl index 4ac73576..3c1ec953 100644 --- a/src/init.jl +++ b/src/init.jl @@ -136,8 +136,18 @@ function __init__() (PyDict, CTryConvertRule_wrapref, 100), ]) C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), (PyBuffer, CTryConvertRule_wrapref, -200), ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 0), + ]) with_gil() do @@ -174,33 +184,33 @@ function __init__() # EXPERIMENTAL: hooks to perform actions when certain modules are loaded if !CONFIG.isembedded - # py""" - # import sys - # class JuliaCompatHooks: - # def __init__(self): - # self.hooks = {} - # def find_module(self, name, path=None): - # hs = self.hooks.get(name) - # if hs is not None: - # for h in hs: - # h() - # def add_hook(self, name, h): - # if name not in self.hooks: - # self.hooks[name] = [h] - # else: - # self.hooks[name].append(h) - # if name in sys.modules: - # h() - # JULIA_COMPAT_HOOKS = JuliaCompatHooks() - # sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) - - # # Before Qt is loaded, fix the path used to look up its plugins - # qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) - # JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) - # JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) - # """ + @py ``` + import sys + class JuliaCompatHooks: + def __init__(self): + self.hooks = {} + def find_module(self, name, path=None): + hs = self.hooks.get(name) + if hs is not None: + for h in hs: + h() + def add_hook(self, name, h): + if name not in self.hooks: + self.hooks[name] = [h] + else: + self.hooks[name].append(h) + if name in sys.modules: + h() + JULIA_COMPAT_HOOKS = JuliaCompatHooks() + sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) + + # Before Qt is loaded, fix the path used to look up its plugins + qtfix_hook = $(() -> (CONFIG.qtfix && fix_qt_plugin_path(); nothing)) + JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) + JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) + JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) + JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) + ``` @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin IJulia.push_postexecute_hook() do diff --git a/src/old/PyArray.jl b/src/old/PyArray.jl deleted file mode 100644 index dcf0dd0c..00000000 --- a/src/old/PyArray.jl +++ /dev/null @@ -1,138 +0,0 @@ -""" - PyArray{T,N,R,M,L}(o) - -Interpret the Python array `o` as a Julia array. - -Type parameters which are not given or set to `missing` are inferred: -- `T` is the (Julia) element type. -- `N` is the number of dimensions. -- `R` is the type of elements of the underlying buffer (which may be different from `T` to allow some basic conversion). -- `M` is true if the array is mutable. -- `L` is true if the array supports fast linear indexing. -""" -mutable struct PyArray{T,N,R,M,L} <: AbstractArray{T,N} - o :: PyObject - ptr :: Ptr{R} - size :: NTuple{N,Int} - length :: Int - bytestrides :: NTuple{N,Int} - handle :: Any -end - -const PyVector{T,R,M,L} = PyArray{T,1,R,M,L} -const PyMatrix{T,R,M,L} = PyArray{T,2,R,M,L} -export PyArray, PyVector, PyMatrix - -function PyArray{T,N,R,M,L}(o::PyObject, info=pyarray_info(o)) where {T,N,R,M,L} - # R - buffer element type - if R === missing - return PyArray{T, N, info.eltype, M, L}(o, info) - elseif R isa Type - Base.allocatedinline(R) || error("source must be allocated inline, got R=$R") - Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R") - else - error("R must be missing or a type") - end - - # T - array element type - if T === missing - return PyArray{pyarray_default_T(R), N, R, M, L}(o, info) - elseif T isa Type - # great - else - error("T must be missing or a type") - end - - # N - if N === missing - return PyArray{T, Int(info.ndims), R, M, L}(o, info) - elseif N isa Int - N == info.ndims || error("source dimension is $(info.ndims), got N=$N") - # great - elseif N isa Integer - return PyArray{T, Int(N), R, M, L}(o, info) - else - error("N must be missing or an integer") - end - - # M - if M === missing - return PyArray{T, N, R, Bool(info.mutable), L}(o, info) - elseif M === true - info.mutable || error("source is immutable, got L=$L") - elseif M === false - # great - else - error("M must be missing, true or false") - end - - bytestrides = NTuple{N, Int}(info.bytestrides) - size = NTuple{N, Int}(info.size) - - # L - if L === missing - return PyArray{T, N, R, M, N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides}(o, info) - elseif L === true - N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable") - elseif L === false - # great - else - error("L must be missing, true or false") - end - - PyArray{T, N, R, M, L}(o, Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle) -end -PyArray{T,N,R,M}(o) where {T,N,R,M} = PyArray{T,N,R,M,missing}(o) -PyArray{T,N,R}(o) where {T,N,R} = PyArray{T,N,R,missing}(o) -PyArray{T,N}(o) where {T,N} = PyArray{T,N,missing}(o) -PyArray{T}(o) where {T} = PyArray{T,missing}(o) -PyArray(o) where {} = PyArray{missing}(o) - -pyobject(x::PyArray) = x.o - -function pyarray_info(o::PyObject) - # TODO: support the numpy array interface too - b = PyBuffer(o, C.PyBUF_RECORDS_RO) - (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b) -end - -Base.isimmutable(x::PyArray{T,N,R,M,L}) where {T,N,R,M,L} = !M -Base.pointer(x::PyArray{T,N,T}) where {T,N} = x.ptr -Base.size(x::PyArray) = x.size -Base.length(x::PyArray) = x.length -Base.IndexStyle(::Type{PyArray{T,N,R,M,L}}) where {T,N,R,M,L} = L ? Base.IndexLinear() : Base.IndexCartesian() - -Base.@propagate_inbounds Base.getindex(x::PyArray{T,N,R,M,L}, i::Vararg{Int,N2}) where {T,N,R,M,L,N2} = - if (N2==N) || (L && N2==1) - @boundscheck checkbounds(x, i...) - pyarray_load(T, x.ptr + pyarray_offset(x, i...)) - else - invoke(getindex, Tuple{AbstractArray{T,N}, Vararg{Int,N2}}, x, i...) - end - -Base.@propagate_inbounds Base.setindex!(x::PyArray{T,N,R,true,L}, v, i::Vararg{Int,N2}) where {T,N,R,L,N2} = - if (N2==N) || (L && N2==1) - @boundscheck checkbounds(x, i...) - pyarray_store!(x.ptr + pyarray_offset(x, i...), convert(T, v)) - x - else - invoke(setindex!, Tuple{AbstractArray{T,N}, typeof(v), Vararg{Int,N2}}, x, v, i...) - end - -pyarray_default_T(::Type{R}) where {R} = R -pyarray_default_T(::Type{CPyObjRef}) = PyObject - -pyarray_load(::Type{T}, p::Ptr{T}) where {T} = unsafe_load(p) -pyarray_load(::Type{T}, p::Ptr{CPyObjRef}) where {T} = (o=unsafe_load(p).ptr; o==C_NULL ? throw(UndefRefError()) : pyconvert(T, pyborrowedobject(o))) - -pyarray_store!(p::Ptr{T}, v::T) where {T} = unsafe_store!(p, v) -pyarray_store!(p::Ptr{CPyObjRef}, v::T) where {T} = (C.Py_DecRef(unsafe_load(p).ptr); unsafe_store!(p, CPyObjRef(pyptr(pyincref!(pyobject(v)))))) - -pyarray_offset(x::PyArray{T,N,R,M,true}, i::Int) where {T,N,R,M} = - N==0 ? 0 : (i-1) * x.bytestrides[1] - -pyarray_offset(x::PyArray{T,1,R,M,true}, i::Int) where {T,R,M} = - (i-1) .* x.bytestrides[1] - -pyarray_offset(x::PyArray{T,N}, i::Vararg{Int,N}) where {T,N} = - sum((i .- 1) .* x.bytestrides) diff --git a/src/utils.jl b/src/utils.jl index b5cdc1b3..e300b032 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -83,6 +83,9 @@ pytypestrdescr(::Type{T}) where {T} = begin T == Float16 ? ("$(c)f2", nothing) : T == Float32 ? ("$(c)f4", nothing) : T == Float64 ? ("$(c)f8", nothing) : + T == Complex{Float16} ? ("$(c)c4", nothing) : + T == Complex{Float32} ? ("$(c)c8", nothing) : + T == Complex{Float64} ? ("$(c)c16", nothing) : T == C.PyObjectRef ? ("|O", nothing) : if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) n = fieldcount(T) @@ -103,6 +106,52 @@ pytypestrdescr(::Type{T}) where {T} = begin end end +pytypestrdescr_to_type(ts::String, descr) = begin + # byte swapped? + bsc = ts[1] + bs = bsc=='<' ? !islittleendian() : bsc=='>' ? islittleendian() : bsc=='|' ? false : error("endianness character not supported: $ts") + bs && error("byte-swapping not supported: $ts") + # element type + etc = ts[2] + if etc == 'b' + sz = parse(Int, ts[3:end]) + sz == sizeof(Bool) && return Bool + error("bool of this size not supported: $ts") + elseif etc == 'i' + sz = parse(Int, ts[3:end]) + sz == 1 && return Int8 + sz == 2 && return Int16 + sz == 4 && return Int32 + sz == 8 && return Int64 + sz == 16 && return Int128 + error("signed int of this size not supported: $ts") + elseif etc == 'u' + sz = parse(Int, ts[3:end]) + sz == 1 && return UInt8 + sz == 2 && return UInt16 + sz == 4 && return UInt32 + sz == 8 && return UInt64 + sz == 16 && return UInt128 + error("unsigned int of this size not supported: $ts") + elseif etc == 'f' + sz = parse(Int, ts[3:end]) + sz == 2 && return Float16 + sz == 4 && return Float32 + sz == 8 && return Float64 + error("float of this size not supported: $ts") + elseif etc == 'c' + sz = parse(Int, ts[3:end]) + sz == 4 && return Complex{Float16} + sz == 8 && return Complex{Float32} + sz == 16 && return Complex{Float64} + error("complex of this size not supported: $ts") + elseif etc == 'O' + return C.PyObjectRef + else + error("type not supported: $ts") + end +end + ### TYPE UTILITIES # Used to signal a Python error from functions that return general Julia objects @@ -129,6 +178,12 @@ tryconvert(::Type{T}, x) where {T} = end CTryConvertRule_wrapref(o, ::Type{S}) where {S} = putresult(S(C.PyObjectRef(o))) +CTryConvertRule_trywrapref(o, ::Type{S}) where {S} = + try + putresult(S(C.PyObjectRef(o))) + catch + 0 + end @generated _typeintersect(::Type{T1}, ::Type{T2}) where {T1,T2} = typeintersect(T1, T2)