diff --git a/juliapy/julia/__init__.py b/juliapy/julia/__init__.py index f1503bf7..543407d3 100644 --- a/juliapy/julia/__init__.py +++ b/juliapy/julia/__init__.py @@ -1,6 +1,6 @@ -_CONFIG = dict() +CONFIG = dict() -def _init_(): +def init(): import os, os.path, sys, ctypes as c, types, shutil, subprocess libpath = os.environ.get('JULIAPY_LIB') if libpath is None: @@ -12,12 +12,12 @@ def _init_(): else: if not os.path.isfile(exepath): raise Exception('JULIAPY_EXE=%s does not exist' % repr(exepath)) - _CONFIG['exepath'] = exepath + CONFIG['exepath'] = exepath libpath = subprocess.run([exepath, '-e', 'import Libdl; print(abspath(Libdl.dlpath("libjulia")))'], stdout=(subprocess.PIPE)).stdout.decode('utf8') else: if not os.path.isfile(libpath): raise Exception('JULIAPY_LIB=%s does not exist' % repr(libpath)) - _CONFIG['libpath'] = libpath + CONFIG['libpath'] = libpath try: d = os.getcwd() os.chdir(os.path.dirname(libpath)) @@ -25,7 +25,7 @@ def _init_(): finally: os.chdir(d) - _CONFIG['lib'] = lib + CONFIG['lib'] = lib lib.jl_init__threading.argtypes = [] lib.jl_init__threading.restype = None lib.jl_init__threading() @@ -33,31 +33,26 @@ def _init_(): lib.jl_eval_string.restype = c.c_void_p res = lib.jl_eval_string( ''' - ENV["PYTHONJL_LIBPTR"] = "{}" - import Python - Python.with_gil() do - Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main) + try + ENV["PYTHONJL_LIBPTR"] = "{}" + import Python + Python.with_gil() do + Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main) + end + catch err + @error "Error loading Python.jl" err=err + rethrow() end '''.format(c.pythonapi._handle).encode('utf8')) if res is None: raise Exception('Python.jl did not start properly. Ensure that the Python package is installed in Julia.') - class Wrapper(types.ModuleType): - - def __getattr__(self, k): - return getattr(self.Main, k) - - def __dir__(self): - return super().__dir__() + self.Main.__dir__() - - sys.modules['julia'].__class__ = Wrapper - -_init_() -del _init_ +init() +del init Core = Main.Core Base = Main.Base Python = Main.Python -def _import(*names): - Main.eval(Base.Meta.parse('import ' + ', '.join(names))) +def newmodule(name): + return Base.Module(Base.Symbol(name)) diff --git a/src/PyArray.jl b/src/PyArray.jl index dcf0dd0c..8e9b013b 100644 --- a/src/PyArray.jl +++ b/src/PyArray.jl @@ -3,7 +3,9 @@ Interpret the Python array `o` as a Julia array. -Type parameters which are not given or set to `missing` are inferred: +The input may be anything supporting the buffer protocol or the numpy array interface. +This includes, `bytes`, `bytearray`, `array.array`, `numpy.ndarray`, `pandas.Series`. + - `T` is the (Julia) element type. - `N` is the number of dimensions. - `R` is the type of elements of the underlying buffer (which may be different from `T` to allow some basic conversion). @@ -11,93 +13,137 @@ Type parameters which are not given or set to `missing` are inferred: - `L` is true if the array supports fast linear indexing. """ mutable struct PyArray{T,N,R,M,L} <: AbstractArray{T,N} - o :: PyObject + ref :: PyRef ptr :: Ptr{R} size :: NTuple{N,Int} length :: Int bytestrides :: NTuple{N,Int} handle :: Any end - const PyVector{T,R,M,L} = PyArray{T,1,R,M,L} const PyMatrix{T,R,M,L} = PyArray{T,2,R,M,L} export PyArray, PyVector, PyMatrix -function PyArray{T,N,R,M,L}(o::PyObject, info=pyarray_info(o)) where {T,N,R,M,L} - # R - buffer element type - if R === missing - return PyArray{T, N, info.eltype, M, L}(o, info) - elseif R isa Type - Base.allocatedinline(R) || error("source must be allocated inline, got R=$R") - Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R") - else - error("R must be missing or a type") - end +ispyreftype(::Type{<:PyArray}) = true +pyptr(x::PyArray) = pyptr(x.ref) +Base.unsafe_convert(::Type{CPyPtr}, x::PyArray) = pyptr(x.ref) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyArray} = CTryConvertRule_trywrapref(o, T) +function PyArray{T,N,R,M,L}(o::PyRef, info) where {T,N,R,M,L} # T - array element type - if T === missing - return PyArray{pyarray_default_T(R), N, R, M, L}(o, info) - elseif T isa Type - # great - else - error("T must be missing or a type") - end + T isa Type || error("T must be a type, got T=$T") - # N - if N === missing - return PyArray{T, Int(info.ndims), R, M, L}(o, info) - elseif N isa Int - N == info.ndims || error("source dimension is $(info.ndims), got N=$N") - # great - elseif N isa Integer - return PyArray{T, Int(N), R, M, L}(o, info) - else - error("N must be missing or an integer") - end + # N - number of dimensions + N isa Integer || error("N must be an integer, got N=$N") + N isa Int || return PyArray{T, Int(N), R, M, L}(o, info) + N == info.ndims || error("source dimension is $(info.ndims), got N=$N") - # M - if M === missing - return PyArray{T, N, R, Bool(info.mutable), L}(o, info) - elseif M === true - info.mutable || error("source is immutable, got L=$L") - elseif M === false - # great - else - error("M must be missing, true or false") - end + # R - buffer element type + R isa Type || error("R must be a type, got R=$R") + Base.allocatedinline(R) || error("source elements must be allocated inline, got R=$R") + Base.aligned_sizeof(R) == info.elsize || error("source elements must have size $(info.elsize), got R=$R") - bytestrides = NTuple{N, Int}(info.bytestrides) - size = NTuple{N, Int}(info.size) + # M - mutable + M isa Bool || error("M must be true or false, got M=$M") + !M || info.mutable || error("source is immutable, got M=$M") - # L - if L === missing - return PyArray{T, N, R, M, N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides}(o, info) - elseif L === true - N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable") - elseif L === false - # great - else - error("L must be missing, true or false") - end + bytestrides = info.bytestrides + size = info.size + + # L - linear indexable + L isa Bool || error("L must be true or false, got L=$L") + !L || N ≤ 1 || size_to_fstrides(bytestrides[1], size...) == bytestrides || error("not linearly indexable, got L=$L") - PyArray{T, N, R, M, L}(o, Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle) + PyArray{T, N, R, M, L}(PyRef(o), Ptr{R}(info.ptr), size, N==0 ? 1 : prod(size), bytestrides, info.handle) end -PyArray{T,N,R,M}(o) where {T,N,R,M} = PyArray{T,N,R,M,missing}(o) -PyArray{T,N,R}(o) where {T,N,R} = PyArray{T,N,R,missing}(o) -PyArray{T,N}(o) where {T,N} = PyArray{T,N,missing}(o) -PyArray{T}(o) where {T} = PyArray{T,missing}(o) -PyArray(o) where {} = PyArray{missing}(o) - -pyobject(x::PyArray) = x.o - -function pyarray_info(o::PyObject) - # TODO: support the numpy array interface too - b = PyBuffer(o, C.PyBUF_RECORDS_RO) - (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b) +PyArray{T,N,R,M}(o::PyRef, info) where {T,N,R,M} = PyArray{T,N,R,M, N≤1 || size_to_fstrides(info.bytestrides[1], info.size...) == info.bytestrides}(o, info) +PyArray{T,N,R}(o::PyRef, info) where {T,N,R} = PyArray{T,N,R, info.mutable}(o, info) +PyArray{T,N}(o::PyRef, info) where {T,N} = PyArray{T,N, info.eltype}(o, info) +PyArray{T}(o::PyRef, info) where {T} = PyArray{T, info.ndims}(o, info) +PyArray{<:Any,N}(o::PyRef, info) where {N} = PyArray{pyarray_default_T(info.eltype), N}(o, info) +PyArray(o::PyRef, info) = PyArray{pyarray_default_T(info.eltype)}(o, info) + +(::Type{A})(o; opts...) where {A<:PyArray} = begin + ref = PyRef(o) + info = pyarray_info(ref; opts...) + info = ( + ndims = Int(info.ndims), + eltype = info.eltype :: Type, + elsize = Int(info.elsize), + mutable = info.mutable :: Bool, + bytestrides = NTuple{Int(info.ndims), Int}(info.bytestrides), + size = NTuple{Int(info.ndims), Int}(info.size), + ptr = Ptr{Cvoid}(info.ptr), + handle = info.handle, + ) + A(ref, info) +end + +function pyarray_info(ref; buffer=true, array=true, copy=true) + if array && pyhasattr(ref, "__array_interface__") + pyconvertdescr(x) = begin + @py ``` + def convert(x): + def fix(x): + a = x[0] + a = (a, a) if isinstance(a, str) else (a[0], a[1]) + b = x[1] + c = x[2] if len(x)>2 else 1 + return (a, b, c) + if x is None or isinstance(x, str): + return x + else: + return [fix(y) for y in x] + $(r::Union{Nothing,String,Vector{Tuple{Tuple{String,String}, PyObject, Int}}}) = convert($x) + ``` + r isa Vector ? [(a, pyconvertdescr(b), c) for (a,b,c) in r] : r + end + ai = pygetattr(ref, "__array_interface__") + pyconvert(Int, ai["version"]) == 3 || error("wrong version") + size = pyconvert(Tuple{Vararg{Int}}, ai["shape"]) + ndims = length(size) + typestr = pyconvert(String, ai["typestr"]) + descr = pyconvertdescr(ai.get("descr")) + eltype = pytypestrdescr_to_type(typestr, descr) + elsize = Base.aligned_sizeof(eltype) + strides = pyconvert(Union{Nothing, Tuple{Vararg{Int}}}, ai.get("strides")) + strides === nothing && (strides = size_to_cstrides(elsize, size...)) + pyis(ai.get("mask"), pynone()) || error("mask not supported") + offset = pyconvert(Union{Nothing, Int}, ai.get("offset")) + offset === nothing && (offset = 0) + data = pyconvert(Union{PyObject, Tuple{UInt, Bool}, Nothing}, ai.get("data")) + if data isa Tuple + ptr = Ptr{Cvoid}(data[1]) + mutable = !data[2] + handle = (ref, ai) + else + buf = PyBuffer(data === nothing ? ref : data) + ptr = buf.buf + mutable = !buf.readonly + handle = (ref, ai, buf) + end + return (ndims=ndims, eltype=eltype, elsize=elsize, mutable=mutable, bytestrides=strides, size=size, ptr=ptr, handle=handle) + end + if array && pyhasattr(ref, "__array_struct__") + # TODO + end + if buffer && C.PyObject_CheckBuffer(ref) + try + b = PyBuffer(ref, C.PyBUF_RECORDS_RO) + return (ndims=b.ndim, eltype=b.eltype, elsize=b.itemsize, mutable=!b.readonly, bytestrides=b.strides, size=b.shape, ptr=b.buf, handle=b) + catch + end + end + if array && copy && pyhasattr(ref, "__array__") + try + return pyarray_info(pycall(PyRef, pygetattr(PyRef, ref, "__array__")); buffer=buffer, array=array, copy=false) + catch + end + end + error("given object does not support the buffer protocol or array interface") end Base.isimmutable(x::PyArray{T,N,R,M,L}) where {T,N,R,M,L} = !M -Base.pointer(x::PyArray{T,N,T}) where {T,N} = x.ptr Base.size(x::PyArray) = x.size Base.length(x::PyArray) = x.length Base.IndexStyle(::Type{PyArray{T,N,R,M,L}}) where {T,N,R,M,L} = L ? Base.IndexLinear() : Base.IndexCartesian() @@ -120,13 +166,23 @@ Base.@propagate_inbounds Base.setindex!(x::PyArray{T,N,R,true,L}, v, i::Vararg{I end pyarray_default_T(::Type{R}) where {R} = R -pyarray_default_T(::Type{CPyObjRef}) = PyObject +pyarray_default_T(::Type{C.PyObjectRef}) = PyObject pyarray_load(::Type{T}, p::Ptr{T}) where {T} = unsafe_load(p) -pyarray_load(::Type{T}, p::Ptr{CPyObjRef}) where {T} = (o=unsafe_load(p).ptr; o==C_NULL ? throw(UndefRefError()) : pyconvert(T, pyborrowedobject(o))) +pyarray_load(::Type{T}, p::Ptr{C.PyObjectRef}) where {T} = begin + o = unsafe_load(p).ptr + isnull(o) && throw(UndefRefError()) + ism1(C.PyObject_Convert(o, T)) && pythrow() + takeresult(T) +end pyarray_store!(p::Ptr{T}, v::T) where {T} = unsafe_store!(p, v) -pyarray_store!(p::Ptr{CPyObjRef}, v::T) where {T} = (C.Py_DecRef(unsafe_load(p).ptr); unsafe_store!(p, CPyObjRef(pyptr(pyincref!(pyobject(v)))))) +pyarray_store!(p::Ptr{C.PyObjectRef}, v::T) where {T} = begin + o = C.PyObject_From(v) + isnull(o) && pythrow() + C.Py_DecRef(unsafe_load(p).ptr) + unsafe_store!(p, C.PyObjectRef(o)) +end pyarray_offset(x::PyArray{T,N,R,M,true}, i::Int) where {T,N,R,M} = N==0 ? 0 : (i-1) * x.bytestrides[1] diff --git a/src/PyBuffer.jl b/src/PyBuffer.jl index 3fae550e..b52a57bc 100644 --- a/src/PyBuffer.jl +++ b/src/PyBuffer.jl @@ -1,5 +1,5 @@ """ - PyBuffer(o) + PyBuffer(o, [flags=C.PyBUF_FULL_RO]) A reference to the underlying buffer of `o`, if it satisfies the buffer protocol. @@ -20,16 +20,15 @@ Has the following properties: """ mutable struct PyBuffer info :: Array{C.Py_buffer, 0} - function PyBuffer(o::PyObject, flags::Integer=C.PyBUF_FULL_RO) + function PyBuffer(o, flags::Integer=C.PyBUF_FULL_RO) info = fill(C.Py_buffer()) check(C.PyObject_GetBuffer(o, pointer(info), flags)) b = new(info) finalizer(b) do b if CONFIG.isinitialized - err = with_gil() do + with_gil(false) do C.PyBuffer_Release(pointer(b.info)) end - check(err) end end b diff --git a/src/PyCode.jl b/src/PyCode.jl new file mode 100644 index 00000000..3e426aee --- /dev/null +++ b/src/PyCode.jl @@ -0,0 +1,21 @@ +mutable struct PyCode + ref :: PyRef + code :: String + filename :: String + mode :: Symbol + PyCode(code::String, filename::String, mode::Symbol) = begin + mode in (:exec, :eval) || error("invalid mode $(repr(mode))") + new(PyRef(), code, filename, mode) + end +end +export PyCode + +ispyreftype(::Type{PyCode}) = true +pyptr(co::PyCode) = begin + ptr = co.ref.ptr + if isnull(ptr) + ptr = co.ref.ptr = C.Py_CompileString(co.code, co.filename, co.mode == :exec ? C.Py_file_input : co.mode == :eval ? C.Py_eval_input : error("invalid mode $(repr(co.mode))")) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyCode) = checknull(pyptr(x)) diff --git a/src/PyDict.jl b/src/PyDict.jl index 370d882d..d4f7aa96 100644 --- a/src/PyDict.jl +++ b/src/PyDict.jl @@ -1,61 +1,104 @@ """ - PyDict{K=PyObject, V=PyObject}(o=pydict()) + PyDict{K=PyObject, V=PyObject}([o]) Wrap the Python dictionary `o` (or anything satisfying the mapping interface) as a Julia dictionary with keys of type `K` and values of type `V`. + +If `o` is not given, an empty dict is created. """ -struct PyDict{K,V} <: AbstractDict{K,V} - o :: PyObject - PyDict{K,V}(o::PyObject) where {K,V} = new{K,V}(o) +mutable struct PyDict{K,V} <: AbstractDict{K,V} + ref :: PyRef + hasbuiltins :: Bool + PyDict{K,V}(o) where {K,V} = new(PyRef(o), false) + PyDict{K,V}() where {K,V} = new(PyRef(), false) end -PyDict{K,V}(o=pydict()) where {K,V} = PyDict{K,V}(pydict(o)) -PyDict{K}(o=pydict()) where {K} = PyDict{K,PyObject}(o) -PyDict(o=pydict()) = PyDict{PyObject}(o) +PyDict{K}(args...) where {K} = PyDict{K,PyObject}(args...) +PyDict(args...) = PyDict{PyObject,PyObject}(args...) export PyDict -pyobject(x::PyDict) = x.o +const pyglobals = PyDict{String}() +export pyglobals -function Base.iterate(x::PyDict{K,V}, it=pyiter(x.o.items())) where {K,V} +ispyreftype(::Type{<:PyDict}) = true +pyptr(x::PyDict) = begin + ptr = x.ref.ptr + if isnull(ptr) + ptr = x.ref.ptr = C.PyDict_New() + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyDict) = checknull(pyptr(x)) + +Base.iterate(x::PyDict{K,V}, it::PyRef) where {K,V} = begin ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing + if !isnull(ptr) + ko = C.PySequence_GetItem(ptr, 0) + isnull(ko) && pythrow() + r = C.PyObject_Convert(ko, K) + C.Py_DecRef(ko) + ism1(r) && pythrow() + k = C.takeresult(K) + vo = C.PySequence_GetItem(ptr, 1) + isnull(vo) && pythrow() + r = C.PyObject_Convert(vo, V) + C.Py_DecRef(vo) + ism1(r) && pythrow() + v = C.takeresult(V) + (k => v, it) + elseif C.PyErr_IsSet() + pythrow() else - kv = pynewobject(ptr) - (pyconvert(K, kv[0]) => pyconvert(V, kv[1])), it + nothing end end +Base.iterate(x::PyDict) = begin + a = C.PyObject_GetAttrString(x, "items") + isnull(a) && pythrow() + b = C.PyObject_CallNice(a) + C.Py_DecRef(a) + isnull(b) && pythrow() + it = C.PyObject_GetIter(b) + C.Py_DecRef(b) + isnull(it) && pythrow() + iterate(x, pynewref(it)) +end -Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = - (pysetitem(x.o, convert(K, k), convert(V, v)); x) +Base.setindex!(x::PyDict{K,V}, v, k) where {K,V} = pysetitem(x, convertref(K, k), convertref(V, v)) -Base.getindex(x::PyDict{K,V}, k) where {K,V} = - pyconvert(V, pygetitem(x.o, convert(K, k))) +Base.getindex(x::PyDict{K,V}, k) where {K,V} = pygetitem(V, x, convertref(K, k)) -Base.delete!(x::PyDict{K,V}, k) where {K,V} = - (pydelitem(x.o, convert(K, k)); x) +Base.delete!(x::PyDict{K}, k) where {K} = pydelitem(x, convertref(K, k)) -Base.length(x::PyDict) = Int(pylen(x.o)) +Base.length(x::PyDict) = Int(pylen(x)) -Base.empty!(x::PyDict) = (x.o.clear(); x) +Base.empty!(x::PyDict) = (@py `$x.clear()`; x) -Base.copy(x::PyDict) = typeof(x)(x.o.copy()) +Base.copy(x::PyDict) = @pyv `$x.copy()`::typeof(x) + +Base.haskey(x::PyDict{K}, _k) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + k === NOTIMPLEMENTED() && return false + pycontains(x, k) +end -function Base.get(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : d +Base.get(x::PyDict{K}, _k, d) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + (k !== NOTIMPLEMENTED() && pycontains(x, k)) ? x[k] : d end -function Base.get(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : f() +Base.get(d::Function, x::PyDict{K}, _k) where {K} = begin + k = tryconvertref(K, _k) + k === PYERR() && pythrow() + (k !== NOTIMPLEMENTED() && pycontains(x, k)) ? x[k] : d() end -function Base.get!(x::PyDict{K,V}, _k, d) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = d) +Base.get!(x::PyDict{K,V}, _k, d) where {K,V} = begin + k = convertref(K, _k) + pycontains(x, k) ? x[k] : (x[k] = convert(V, d)) end -function Base.get!(f::Function, x::PyDict{K,V}, _k) where {K,V} - k = convert(K, _k) - pycontains(x.o, k) ? x[k] : (x[k] = f()) +Base.get!(d::Function, x::PyDict{K,V}, _k) where {K,V} = begin + k = convertref(K, _k) + pycontains(x, k) ? x[k] : (x[k] = convert(V, d())) end diff --git a/src/PyException.jl b/src/PyException.jl new file mode 100644 index 00000000..6a7075bb --- /dev/null +++ b/src/PyException.jl @@ -0,0 +1,171 @@ +mutable struct PyException <: Exception + tref :: PyRef + vref :: PyRef + bref :: PyRef + PyException(::Val{:new}, t::Ptr=C_NULL, v::Ptr=C_NULL, b::Ptr=C_NULL, borrowed::Bool=false) = + new(PyRef(Val(:new), t, borrowed), PyRef(Val(:new), v, borrowed), PyRef(Val(:new), b, borrowed)) +end +export PyException + +pythrow() = throw(PyException(Val(:new), C.PyErr_FetchTuple(true)...)) + +""" + check(x) + +Checks if the Python error indicator is set. If so, throws the current Python exception. Otherwise returns `x`. +""" +check(x) = C.PyErr_IsSet() ? pythrow() : x + +""" + checkm1(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == -1` instead. + +If `ambig` is true the error indicator is also checked. +""" +checkm1(x::Number, ambig::Bool=false) = ism1(x) ? ambig ? check(x) : pythrow() : x + +""" + checknull(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == C_NULL` instead. + +If `ambig` is true the error indicator is also checked. +""" +checknull(x::Ptr, ambig::Bool=false) = isnull(x) ? ambig ? check(x) : pythrow() : x + +""" + checkerr(x, ambig=false) + +Same as `check(x)` but errors are indicated by `x == PYERR()` instead. + +If `ambig` is true the error indicator is also checked. +""" +checkerr(x, ambig::Bool=false) = x===PYERR() ? ambig ? check(x) : pythrow() : x + +""" + checknullconvert(T, x, ambig=false) :: T + +Same as `checknull(x, ambig)` but steals a reference to `x` and converts the result to a `T`. +""" +checknullconvert(::Type{T}, x::Ptr, ambig::Bool=false) where {T} = begin + if isnull(x) && (!ambig || C.PyErr_IsSet()) + C.Py_DecRef(x) + pythrow() + end + r = C.PyObject_Convert(x, T) + C.Py_DecRef(x) + checkm1(r) + C.takeresult(T) +end + +function Base.showerror(io::IO, e::PyException) + if isnull(e.tref) + print(io, "Python: mysterious error (no error was actually set)") + return + end + + if CONFIG.sysautolasttraceback + err = C.Py_DecRef(C.PyImport_ImportModule("sys"), PYERR()) do sys + err = C.PyObject_SetAttrString(sys, "last_type", isnull(e.tref) ? C.Py_None() : e.tref.ptr) + ism1(err) && return PYERR() + err = C.PyObject_SetAttrString(sys, "last_value", isnull(e.vref) ? C.Py_None() : e.vref.ptr) + ism1(err) && return PYERR() + err = C.PyObject_SetAttrString(sys, "last_traceback", isnull(e.bref) ? C.Py_None() : e.bref.ptr) + ism1(err) && return PYERR() + nothing + end + if err == PYERR() + C.PyErr_Clear() + print(io, "") + end + end + + # if this is a Julia exception then recursively print it and its stacktrace + if C.PyErr_GivenExceptionMatches(e.tref, C.PyExc_JuliaError()) != 0 + try + # Extract error value + vo = @pyv `$(e.vref).args[0]`::Any + if vo isa Tuple{Exception, Any} + je, jb = vo + else + je = vo + jb = nothing + end + print(io, "Python: Julia: ") + # Show exception + if je isa Exception + showerror(io, je) + else + print(io, je) + end + # Show backtrace + if jb === nothing + println(io) + print(io, "Stacktrace: none") + else + io2 = IOBuffer() + Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) + printstyled(io, String(take!(io2))) + end + # Show Python backtrace + if isnull(e.bref) + println(io) + printstyled(io, "Python stacktrace: none") + return + else + @goto pystacktrace + end + catch err + println("") + end + end + + # otherwise, print the Python exception + print(io, "Python: ") + + # print the type name + try + tname = @pyv `$(e.tref).__name__`::String + print(io, tname) + catch + print(io, "") + end + + # print the error message + if !isnull(e.vref) + print(io, ": ") + try + vstr = @pyv `str($(e.vref))`::String + print(io, vstr) + catch + print(io, "") + end + end + + # print the stacktrace + if !isnull(e.bref) + @label pystacktrace + println(io) + printstyled(io, "Python stacktrace:") + try + @py ``` + import traceback + $(fs :: Vector{Tuple{String, String, Int}}) = [(x.name, x.filename, x.lineno) for x in traceback.extract_tb($(e.bref))] + ``` + for (i,(name, fname, lineno)) in enumerate(fs) + println(io) + printstyled(io, " [", i, "] ") + printstyled(io, name, bold=true) + printstyled(io, " at ") + # if (m=match(r"^(.*):([0-9]+)$", fname)) !== nothing + # fname = m.captures[1] + # lineno += parse(Int, m.captures[2]) - 1 + # end + printstyled(io, fname, ":", lineno, bold=true) + end + catch err + print(io, "") + end + end +end diff --git a/src/PyIO.jl b/src/PyIO.jl index 7d777139..b1c719f0 100644 --- a/src/PyIO.jl +++ b/src/PyIO.jl @@ -10,7 +10,7 @@ If `text=false` then `o` must be a binary stream and arbitrary binary I/O is pos For efficiency, reads and writes are buffered before being sent to `o`. The size of the buffer is `buflen`. """ mutable struct PyIO <: IO - o :: PyObject + ref :: PyRef # true to close the file automatically own :: Bool # true if `o` is text, false if binary @@ -24,8 +24,8 @@ mutable struct PyIO <: IO obuflen :: Int obuf :: Vector{UInt8} - function PyIO(o::PyObject; own::Bool=false, text::Union{Missing,Bool}=missing, buflen::Integer=4096, ibuflen=buflen, obuflen=buflen) - io = new(PyObject(o), own, text===missing ? pyisinstance(o, pyiomodule.TextIOBase) : text, false, ibuflen, UInt8[], obuflen, UInt8[]) + function PyIO(o; own::Bool=false, text::Union{Missing,Bool}=missing, buflen::Integer=4096, ibuflen=buflen, obuflen=buflen) + io = new(PyRef(o), own, text===missing ? pyisinstance(o, pyiomodule().TextIOBase) : text, false, ibuflen, UInt8[], obuflen, UInt8[]) finalizer(io) do io io.own ? close(io) : flush(io) end @@ -34,6 +34,11 @@ mutable struct PyIO <: IO end export PyIO +ispyreftype(::Type{PyIO}) = true +pyptr(io::PyIO) = pyptr(io.ref) +Base.unsafe_convert(::Type{CPyPtr}, io::PyIO) = checknull(pyptr(io)) +C.PyObject_TryConvert__initial(o, ::Type{PyIO}) = C.putresult(PyIO(pyborrowedref(o))) + """ PyIO(f, o; ...) @@ -50,12 +55,10 @@ function PyIO(f::Function, o; opts...) end end -pyobject(io::PyIO) = io.o - # If obuf is non-empty, write it to the underlying stream. function putobuf(io::PyIO) if !isempty(io.obuf) - io.text ? io.o.write(String(io.obuf)) : io.o.write(io.obuf) + @py `$io.write($(io.text ? pystr(io.obuf) : pybytes(io.obuf)))` empty!(io.obuf) end nothing @@ -63,26 +66,21 @@ end # If ibuf is empty, read some more from the underlying stream. # After this call, if ibuf is empty then we are at EOF. +# TODO: in binary mode, `io.readinto()` to avoid copying data function getibuf(io::PyIO) if isempty(io.ibuf) - if io.text - append!(io.ibuf, PyVector{UInt8,UInt8,false,true}(pystr_asutf8string(io.o.read(io.ibuflen)))) - else - resize!(io.ibuf, io.ibuflen) - n = io.o.readinto(io.ibuf).jl!i - resize!(io.ibuf, n) - end + append!(io.ibuf, @pyv `$io.read($(io.ibuflen))`::Vector{UInt8}) end nothing end function Base.flush(io::PyIO) putobuf(io) - io.o.flush() + @py `$io.flush()` nothing end -Base.close(io::PyIO) = (flush(io); io.o.close(); nothing) +Base.close(io::PyIO) = (flush(io); @py `$io.close()`; nothing) function Base.eof(io::PyIO) if io.eof @@ -95,13 +93,13 @@ function Base.eof(io::PyIO) end end -Base.fd(io::PyIO) = io.o.fileno().jl!i +Base.fd(io::PyIO) = @pyv `$io.fileno()`::Int -Base.isreadable(io::PyIO) = io.o.readable().jl!b +Base.isreadable(io::PyIO) = @pyv `$io.readable()`::Bool -Base.iswritable(io::PyIO) = io.o.writable().jl!b +Base.iswritable(io::PyIO) = @pyv `$io.writable()`::Bool -Base.isopen(io::PyIO) = !io.o.closed.jl!b +Base.isopen(io::PyIO) = @pyv `not $io.closed`::Bool function Base.unsafe_write(io::PyIO, ptr::Ptr{UInt8}, n::UInt) while true @@ -150,7 +148,7 @@ function Base.seek(io::PyIO, pos::Integer) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(pos, 0) + @py `$io.seek($pos, 0)` io end @@ -164,7 +162,7 @@ function Base.seekstart(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(0, 0) + @py `$io.seek(0, 0)` io end @@ -172,7 +170,7 @@ function Base.seekend(io::PyIO) putobuf(io) empty!(io.ibuf) io.eof = false - io.o.seek(0, 2) + @py `$io.seek(0, 2)` io end @@ -188,7 +186,7 @@ function Base.skip(io::PyIO, n::Integer) if 0 ≤ n ≤ io.ibuflen read(io, n) else - io.o.seek(n - length(io.ibuf), 1) + @py `$io.seek($(n - length(io.ibuf)), 1)` empty!(io.ibuf) io.eof = false end @@ -199,11 +197,11 @@ function Base.position(io::PyIO) putobuf(io) if io.text if isempty(io.ibuf) - io.o.position().jl!i + @pyv `$io.position()`::Int else error("`position(io)` text PyIO streams only implemented for empty input buffer (e.g. do `read(io, length(io.ibuf))` first)") end else - io.o.position().jl!i - length(io.ibuf) + (@py `$io.position()`::Int) - length(io.ibuf) end end diff --git a/src/PyInternedString.jl b/src/PyInternedString.jl new file mode 100644 index 00000000..f4baaec5 --- /dev/null +++ b/src/PyInternedString.jl @@ -0,0 +1,29 @@ +mutable struct PyInternedString + ref :: PyRef + val :: String + PyInternedString(x::String) = new(PyRef(), x) +end + +ispyreftype(::Type{PyInternedString}) = true +pyptr(x::PyInternedString) = begin + ptr = x.ref.ptr + if isnull(ptr) + s = Ref{CPyPtr}() + s[] = C.PyUnicode_From(x.val) + isnull(s[]) && return ptr + C.PyUnicode_InternInPlace(s) + ptr = x.ref.ptr = s[] + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyInternedString) = checknull(pyptr(x)) + +""" + pystr"..." + +An interned Python string. +""" +macro pystr_str(s::String) + PyInternedString(s) +end +export @pystr_str diff --git a/src/PyIterable.jl b/src/PyIterable.jl index f7a9585d..00f863b8 100644 --- a/src/PyIterable.jl +++ b/src/PyIterable.jl @@ -4,15 +4,18 @@ Wrap the Python object `o` into a Julia object which iterates values of type `T`. """ struct PyIterable{T} - o :: PyObject - PyIterable{T}(o) where {T} = new{T}(o) + ref :: PyRef + PyIterable{T}(o) where {T} = new{T}(PyRef(o)) end PyIterable(o) = PyIterable{PyObject}(o) export PyIterable -pyobject(x::PyIterable) = x.o +ispyreftype(::Type{<:PyIterable}) = true +pyptr(x::PyIterable) = pyptr(x.ref) +Base.unsafe_convert(::Type{CPyPtr}, x::PyIterable) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyIterable} = C.putresult(T(pyborrowedref(o))) -Base.length(x::PyIterable) = Int(pylen(x.o)) +Base.length(x::PyIterable) = Int(pylen(x)) Base.IteratorSize(::Type{<:PyIterable}) = Base.SizeUnknown() @@ -20,12 +23,16 @@ Base.IteratorEltype(::Type{<:PyIterable}) = Base.HasEltype() Base.eltype(::Type{PyIterable{T}}) where {T} = T -function Base.iterate(x::PyIterable{T}, it=pyiter(x.o)) where {T} - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing +function Base.iterate(x::PyIterable{T}, it=pyiter(PyRef, x)) where {T} + vo = C.PyIter_Next(it) + if !isnull(vo) + r = C.PyObject_Convert(vo, T) + C.Py_DecRef(vo) + checkm1(r) + (takeresult(T), it) + elseif C.PyErr_IsSet() + pythrow() else - (pyconvert(T, pynewobject(ptr)), it) + nothing end end diff --git a/src/PyList.jl b/src/PyList.jl index 47cbfa2d..ca890bc5 100644 --- a/src/PyList.jl +++ b/src/PyList.jl @@ -1,43 +1,75 @@ """ - PyList{T=PyObject}(o=pylist()) + PyList{T=PyObject}([o]) Wrap the Python list `o` (or anything satisfying the sequence interface) as a Julia vector with elements of type `T`. + +If `o` is not given, an empty list is created. """ struct PyList{T} <: AbstractVector{T} - o :: PyObject - PyList{T}(o::PyObject) where {T} = new{T}(o) + ref :: PyRef + PyList{T}(o) where {T} = new{T}(PyRef(o)) + PyList{T}() where {T} = new{T}(PyRef()) end -PyList{T}(o=pylist()) where {T} = PyList{T}(pylist(o)) -PyList(o=pylist()) = PyList{PyObject}(o) +PyList(o) = PyList{PyObject}(o) +PyList() = PyList{PyObject}() export PyList -pyobject(x::PyList) = x.o +ispyreftype(::Type{<:PyList}) = true +pyptr(x::PyList) = begin + ptr = x.ref.ptr + if isnull(ptr) + ptr = x.ref.ptr = C.PyList_New(0) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PyList) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PyList} = C.putresult(T(pyborrowedref(o))) -Base.length(x::PyList) = Int(pylen(x.o)) +# Base.length(x::PyList) = @pyv `len($x)`::Int +Base.length(x::PyList) = Int(pylen(x)) Base.size(x::PyList) = (length(x),) -Base.getindex(x::PyList{T}, i::Integer) where {T} = pyconvert(T, pygetitem(x.o, i-1)) +Base.getindex(x::PyList{T}, i::Integer) where {T} = begin + checkbounds(x, i) + # The following line implements this function, but is typically 3x slower. + # @pyv `$x[$(i-1)]`::T + p = checknull(C.PySequence_GetItem(x, i-1)) + r = C.PyObject_Convert(p, T) + C.Py_DecRef(p) + checkm1(r) + C.takeresult(T) +end -Base.setindex!(x::PyList{T}, v, i::Integer) where {T} = (pysetitem(x.o, i-1, convert(T, v)); x) +Base.setindex!(x::PyList{T}, _v, i::Integer) where {T} = begin + checkbounds(x, i) + v = convertref(T, _v) + # The following line implements this function, but is typically 10x slower + # @py `$x[$(i-1)] = $v` + vp = checknull(C.PyObject_From(v)) + err = C.PySequence_SetItem(x, i-1, vp) + C.Py_DecRef(vp) + checkm1(err) + x +end -Base.insert!(x::PyList{T}, i::Integer, v) where {T} = (x.o.insert(i-1, convert(T, v)); x) +Base.insert!(x::PyList{T}, i::Integer, v) where {T} = (i==length(x)+1 || checkbounds(x, i); @py `$x.insert($(i-1), $(convertref(T, v)))`; x) -Base.push!(x::PyList{T}, v) where {T} = (x.o.append(convert(T, v)); x) +Base.push!(x::PyList{T}, v) where {T} = (@py `$x.append($(convertref(T, v)))`; x) Base.pushfirst!(x::PyList, v) = insert!(x, 1, v) -Base.pop!(x::PyList{T}) where {T} = pyconvert(T, x.o.pop()) +Base.pop!(x::PyList{T}) where {T} = @pyv `$x.pop()`::T -Base.popat!(x::PyList{T}, i::Integer) where {T} = pyconvert(T, x.o.pop(i-1)) +Base.popat!(x::PyList{T}, i::Integer) where {T} = (checkbounds(x, i); @pyv `$x.pop($(i-1))`::T) Base.popfirst!(x::PyList) = pop!(x, 1) -Base.reverse!(x::PyList) = (x.o.reverse(); x) +Base.reverse!(x::PyList) = (@py `$x.reverse()`; x) # TODO: support kwarg `by` (becomes python kwarg `key`) -Base.sort!(x::PyList; rev=false) = (x.o.sort(reverse=rev); x) +Base.sort!(x::PyList; rev::Bool=false) = (@py `$x.sort(reverse=$rev)`; x) -Base.empty!(x::PyList) = (x.o.clear(); x) +Base.empty!(x::PyList) = (@py `$x.clear()`; x) -Base.copy(x::PyList) = typeof(x)(x.o.copy()) +Base.copy(x::PyList) = @pyv `$x.copy()`::typeof(x) diff --git a/src/PyObject.jl b/src/PyObject.jl new file mode 100644 index 00000000..306a64f0 --- /dev/null +++ b/src/PyObject.jl @@ -0,0 +1,356 @@ +mutable struct PyObject + ref :: PyRef + make :: Any + PyObject(::Val{:nocopy}, o::PyRef) = new(o) + PyObject(o) = new(PyRef(o)) + PyObject(::Val{:lazy}, mk) = new(PyRef(), mk) +end +export PyObject + +ispyreftype(::Type{PyObject}) = true +pyptr(o::PyObject) = begin + ref = getfield(o, :ref) + ptr = ref.ptr + if isnull(ptr) + val = try + getfield(o, :make)() + catch err + C.PyErr_SetString(C.PyExc_Exception(), "Error retrieving object value: $err") + return ptr + end + ptr = ref.ptr = C.PyObject_From(val) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, o::PyObject) = checknull(pyptr(o)) +pynewobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyObject(Val(:nocopy), pynewref(p)) +pyborrowedobject(p::Ptr, check::Bool=false) = (check && isnull(p)) ? pythrow() : PyObject(Val(:nocopy), pyborrowedref(p)) +pylazyobject(mk) = PyObject(Val(:lazy), mk) + +C.PyObject_TryConvert__initial(o, ::Type{PyObject}) = C.putresult(pyborrowedobject(o)) + +Base.convert(::Type{PyObject}, x::PyObject) = x +Base.convert(::Type{Any}, x::PyObject) = x +Base.convert(::Type{T}, x::PyObject) where {T} = x isa T ? x : pyconvert(T, x) +Base.convert(::Type{PyObject}, x) = PyObject(x) + +### Cache some common values + +const _pynone = pylazyobject(() -> pynone(PyRef)) +pynone(::Type{PyObject}) = _pynone + +const _pytrue = pylazyobject(() -> pybool(PyRef, true)) +const _pyfalse = pylazyobject(() -> pybool(PyRef, false)) +pybool(::Type{PyObject}, x::Bool) = x ? _pytrue : _pyfalse + +### IO + +Base.string(o::PyObject) = pystr(String, o) + +Base.print(io::IO, o::PyObject) = print(io, string(o)) + +Base.show(io::IO, o::PyObject) = begin + s = pyrepr(String, o) + if get(io, :typeinfo, Any) == typeof(o) + print(io, s) + elseif startswith(s, "<") && endswith(s, ">") + print(io, "") + end +end + +function Base.show(io::IO, ::MIME"text/plain", o::PyObject) + prefix = get(io, :typeinfo, Any) != typeof(o) + h, w = displaysize(io) + h -= 3 + x = try + pystr(String, pypprintmodule().pformat(o, width=w)) + catch + pyrepr(String, o) + end + if get(io, :limit, true) + if '\n' ∈ x + # multiple lines + # each one is truncated to one screen width + if prefix + print(io, "py:") + h -= 1 + end + xs = split(x, '\n') + printlines(xs, nl=true) = + for (i,x) in enumerate(xs) + (nl || i>1) && print(io, '\n') + if length(x) ≤ w + print(io, x) + else + print(io, x[1:nextind(x, 0, w-1)], '…') + end + end + if length(xs) ≤ h + # all lines fit on screen + printlines(xs, prefix) + else + # too many lines, skip the middle ones + h -= 1 + h2 = cld(h, 2) + h3 = (length(xs)+1)-(h-h2) + printlines(xs[1:h2], prefix) + linelen = min( + checkbounds(Bool, xs, h2) ? length(xs[h2]) : 0, + checkbounds(Bool, xs, h3) ? length(xs[h3]) : 0, + w + ) + msg = "... [skipping $(h3-h2-1) lines] ..." + pad = fld(linelen - length(msg), 2) + print(io, "\n", pad > 0 ? " "^pad : "", msg) + printlines(xs[h3:end]) + end + elseif length(x) ≤ (prefix ? w-4 : w) + # one short line + print(io, prefix ? "py: " : "", x) + return + else + # one long line + if prefix + println(io, "py:") + h -= 1 + end + a = h * w + if length(x) ≤ a + # whole string fits on screen + print(io, x) + else + # too long, skip the middle + h -= 1 + h2 = cld(h, 2) + i2 = nextind(x, 0, h2 * w) + i3 = prevind(x, ncodeunits(x)+1, (h-h2)*w) + println(io, x[1:i2]) + println(io, " ... [skipping $(length(x[nextind(x,i2):prevind(x,i3)])) characters] ...") + print(io, x[i3:end]) + end + end + else + print(io, prefix ? "py: " : "", x) + end +end + +Base.show(io::IO, mime::_py_mimetype, o::PyObject) = _py_mime_show(io, mime, o) +Base.showable(mime::_py_mimetype, o::PyObject) = _py_mime_showable(mime, o) + +### PROPERTIES + +Base.getproperty(o::PyObject, k::Symbol) = + if k == :jl! + ((::Type{T}) where {T}) -> pyconvert(T, o) + elseif k == :jl!i + pyconvert(Int, o) + elseif k == :jl!b + pytruth(o) + elseif k == :jl!s + pystr(String, o) + elseif k == :jl!r + pyrepr(String, o) + elseif k == :jl!f + pyconvert(Cdouble, o) + elseif k == :jl!c + pyconvert(Complex{Cdouble}, o) + elseif k == :jl!iter + ((::Type{T}=PyObject) where {T}) -> PyIterable{T}(o) + elseif k == :jl!list + ((::Type{T}=PyObject) where {T}) -> PyList{T}(o) + elseif k == :jl!set + ((::Type{T}=PyObject) where {T}) -> PySet{T}(o) + elseif k == :jl!dict + ((::Type{K}=PyObject, ::Type{V}=PyObject) where {K,V}) -> PyDict{K,V}(o) + elseif k == :jl!io + (; opts...) -> PyIO(o; opts...) + elseif k == :jl!pandasdf + (; opts...) -> PyPandasDataFrame(o; opts...) + else + pygetattr(PyObject, o, k) + end + +Base.setproperty!(o::PyObject, k::Symbol, v) = pysetattr(o, k, v) + +Base.hasproperty(o::PyObject, k::Symbol) = pyhasattr(o, k) + +function Base.propertynames(o::PyObject) + # this follows the logic of rlcompleter.py + @py ``` + def members(o): + def classmembers(c): + r = dir(c) + if hasattr(c, '__bases__'): + for b in c.__bases__: + r += classmembers(b) + return r + words = set(dir(o)) + words.discard('__builtins__') + if hasattr(o, '__class__'): + words.add('__class__') + words.update(classmembers(o.__class__)) + return words + $(words::Vector{Symbol}) = members($o) + ``` + words +end + +### CALL + +(f::PyObject)(args...; kwargs...) = pycall(PyObject, f, args...; kwargs...) + +### ITERATE & INDEX + +Base.IteratorSize(::Type{PyObject}) = Base.SizeUnknown() +Base.eltype(::Type{PyObject}) = PyObject + +Base.getindex(o::PyObject, k) = pygetitem(PyObject, o, k) +Base.getindex(o::PyObject, k...) = pygetitem(PyObject, o, k) +Base.setindex!(o::PyObject, v, k) = pysetitem(o, k, v) +Base.setindex!(o::PyObject, v, k...) = pysetitem(o, k, v) +Base.delete!(o::PyObject, k) = pydelitem(o, k) +Base.delete!(o::PyObject, k...) = pydelitem(o, k) + +Base.length(o::PyObject) = Int(pylen(o)) + +Base.iterate(o::PyObject, it::PyRef=pyiter(PyRef,o)) = begin + vo = C.PyIter_Next(it) + if !isnull(vo) + (pynewobject(vo), it) + elseif C.PyErr_IsSet() + pythrow() + else + nothing + end +end + +Base.in(x, o::PyObject) = pycontains(o, x) +Base.hash(o::PyObject) = trunc(UInt, pyhash(o)) + +### COMPARISON + +Base.:(==)(x::PyObject, y::PyObject) = pyeq(PyObject, x, y) +Base.:(!=)(x::PyObject, y::PyObject) = pynq(PyObject, x, y) +Base.:(<=)(x::PyObject, y::PyObject) = pyle(PyObject, x, y) +Base.:(< )(x::PyObject, y::PyObject) = pylt(PyObject, x, y) +Base.:(>=)(x::PyObject, y::PyObject) = pyge(PyObject, x, y) +Base.:(> )(x::PyObject, y::PyObject) = pygt(PyObject, x, y) +Base.isequal(x::PyObject, y::PyObject) = pyeq(Bool, x, y) +Base.isless(x::PyObject, y::PyObject) = pylt(Bool, x, y) + +### ARITHMETIC + +Base.zero(::Type{PyObject}) = pyint(0) +Base.one(::Type{PyObject}) = pyint(1) + +# unary +Base.:(-)(o::PyObject) = pyneg(PyObject, o) +Base.:(+)(o::PyObject) = pypos(PyObject, o) +Base.abs(o::PyObject) = pyabs(PyObject, o) +Base.:(~)(o::PyObject) = pyinv(PyObject, o) + +# binary +Base.:(+)(o1::PyObject, o2::PyObject) = pyadd(PyObject, o1, o2) +Base.:(-)(o1::PyObject, o2::PyObject) = pysub(PyObject, o1, o2) +Base.:(*)(o1::PyObject, o2::PyObject) = pymul(PyObject, o1, o2) +Base.:(/)(o1::PyObject, o2::PyObject) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::PyObject, o2::PyObject) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::PyObject, o2::PyObject) = pymod(PyObject, o1, o2) +Base.:(^)(o1::PyObject, o2::PyObject) = pypow(PyObject, o1, o2) +Base.:(<<)(o1::PyObject, o2::PyObject) = pylshift(PyObject, o1, o2) +Base.:(>>)(o1::PyObject, o2::PyObject) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::PyObject, o2::PyObject) = pyand(PyObject, o1, o2) +Base.:xor(o1::PyObject, o2::PyObject) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::PyObject, o2::PyObject) = pyor(PyObject, o1, o2) + +Base.:(+)(o1::PyObject, o2::Number) = pyadd(PyObject, o1, o2) +Base.:(-)(o1::PyObject, o2::Number) = pysub(PyObject, o1, o2) +Base.:(*)(o1::PyObject, o2::Number) = pymul(PyObject, o1, o2) +Base.:(/)(o1::PyObject, o2::Number) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::PyObject, o2::Number) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::PyObject, o2::Number) = pymod(PyObject, o1, o2) +Base.:(^)(o1::PyObject, o2::Number) = pypow(PyObject, o1, o2) +Base.:(<<)(o1::PyObject, o2::Number) = pylshift(PyObject, o1, o2) +Base.:(>>)(o1::PyObject, o2::Number) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::PyObject, o2::Number) = pyand(PyObject, o1, o2) +Base.:xor(o1::PyObject, o2::Number) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::PyObject, o2::Number) = pyor(PyObject, o1, o2) + +Base.:(+)(o1::Number, o2::PyObject) = pyadd(PyObject, o1, o2) # Defining +(::Any, ::PyObject) like this hangs Julia v1.5.2-v1.5.3 (at least) during precompilation +Base.:(-)(o1::Number, o2::PyObject) = pysub(PyObject, o1, o2) +Base.:(*)(o1::Number, o2::PyObject) = pymul(PyObject, o1, o2) +Base.:(/)(o1::Number, o2::PyObject) = pytruediv(PyObject, o1, o2) +Base.:fld(o1::Number, o2::PyObject) = pyfloordiv(PyObject, o1, o2) +Base.:mod(o1::Number, o2::PyObject) = pymod(PyObject, o1, o2) +# Base.:(^)(o1::Number, o2::PyObject) = pypow(PyObject, o1, o2) +# Base.:(<<)(o1::Number, o2::PyObject) = pylshift(PyObject, o1, o2) +# Base.:(>>)(o1::Number, o2::PyObject) = pyrshift(PyObject, o1, o2) +Base.:(&)(o1::Number, o2::PyObject) = pyand(PyObject, o1, o2) +Base.:xor(o1::Number, o2::PyObject) = pyxor(PyObject, o1, o2) +Base.:(|)(o1::Number, o2::PyObject) = pyor(PyObject, o1, o2) + +# ternary +Base.powermod(o1::PyObject, o2::Union{PyObject,Number}, o3::Union{PyObject,Number}) = pypow(PyObject, o1, o2, o3) + +### DOCUMENTATION + +# function Base.Docs.getdoc(o::PyObject) +# function tryget(f, g=identity) +# a = try +# f() +# catch +# return nothing +# end +# pyisnone(a) ? nothing : g(a) +# end +# # function name(o) +# # n = tryget(()->o.__name__) +# # n === nothing && return nothing +# # m = tryget(()->o.__module__) +# # (m === nothing || string(m) == "builtins") ? string(n) : string(m, ".", n) +# # end +# getname(o) = tryget(()->o.__name__, string) +# getdoc(o) = tryget(()->o.__doc__, string ∘ ins.cleandoc) + +# docs = [] + +# # Short description +# ins = pyimport("inspect") +# desc, name = if ins.ismodule(o).jl!b +# (pyhasattr(o, "__path__") ? "package" : "module"), gname(o) +# elseif ins.isgetsetdescriptor(o).jl!b +# ("getset descriptor", "$(getname(o.__objclass__)).$(o.__name__)") +# elseif ins.ismemberdescriptor(o).jl!b +# ("member descriptor", "$(getname(o.__objclass__)).$(o.__name__)") +# elseif ins.isclass(o).jl!b +# ("class", getname(o)) +# elseif ins.isfunction(o).jl!b || ins.isbuiltin(o).jl!b +# ("function", getname(o)) +# elseif ins.ismethod(o).jl!b +# ("method", getname(o)) +# else +# ("object of type", getname(pytype(o))) +# end +# push!(docs, Markdown.Paragraph(["Python ", desc, " ", Markdown.Code(name), "."])) + +# if ins.isroutine(o).jl!b || ins.isclass(o).jl!b +# try +# push!(docs, Markdown.Code("python", "$(o.__name__)$(ins.signature(o))")) +# catch +# end +# end + +# # Maybe document the class instead +# doc = getdoc(o) +# if doc === nothing && !ins.ismodule(o).jl!b && !ins.isclass(o).jl!b && !ins.isroutine(o).jl!b && !ins.isdatadescriptor(o).jl!b +# o = pyhasattr(o, "__origin__") ? o.__origin__ : pytype(o) +# doc = getdoc(o) +# end +# doc === nothing || push!(docs, Markdown.Paragraph([Markdown.Text(doc)])) + +# # Done +# Markdown.MD(docs) +# end +# Base.Docs.Binding(o::PyObject, k::Symbol) = getproperty(o, k) diff --git a/src/PyObjectArray.jl b/src/PyObjectArray.jl index 3365ab2d..d6ad90fd 100644 --- a/src/PyObjectArray.jl +++ b/src/PyObjectArray.jl @@ -1,10 +1,18 @@ +""" + PyObjectArray(undef, dims...) + PyObjectArray(array) + +An array of `PyObject`s which supports the Python buffer protocol. + +Internally, the objects are stored as an array of pointers. +""" mutable struct PyObjectArray{N} <: AbstractArray{PyObject, N} ptrs :: Array{CPyPtr, N} function PyObjectArray{N}(::UndefInitializer, dims::NTuple{N,Integer}) where {N} x = new{N}(fill(CPyPtr(C_NULL), dims)) finalizer(x) do x if CONFIG.isinitialized - with_gil() do + with_gil(false) do for ptr in x.ptrs C.Py_DecRef(ptr) end @@ -34,10 +42,11 @@ function Base.getindex(x::PyObjectArray, i::Integer...) end function Base.setindex!(x::PyObjectArray, _v, i::Integer...) - v = convert(PyObject, _v) + vo = C.PyObject_From(_v) + isnull(vo) && pythrow() C.Py_DecRef(x.ptrs[i...]) - pyincref!(v) - x.ptrs[i...] = pyptr(v) + C.Py_IncRef(vo) + x.ptrs[i...] = vo x end @@ -46,3 +55,15 @@ function Base.deleteat!(x::PyObjectArray, i::Integer) deleteat!(x.ptrs, i) x end + +C._pyjlarray_get_buffer(o, buf, flags, x::PyObjectArray) = + C.pyjl_get_buffer_impl(o, buf, flags, pointer(x.ptrs), sizeof(CPyPtr), length(x), ndims(x), "O", size(x), strides(x.ptrs), true) + +C._pyjlarray_array_interface(x::PyObjectArray) = + C.PyDict_From(Dict( + "shape" => size(x), + "typestr" => "|O", + "data" => (UInt(pointer(x.ptrs)), false), + "strides" => strides(x.ptrs) .* sizeof(CPyPtr), + "version" => 3, + )) diff --git a/src/PyPandasDataFrame.jl b/src/PyPandasDataFrame.jl new file mode 100644 index 00000000..590471b3 --- /dev/null +++ b/src/PyPandasDataFrame.jl @@ -0,0 +1,111 @@ +asvector(x::AbstractVector) = x +asvector(x) = collect(x) + +""" + pycolumntable([T=PyObject,] src) :: T + +Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. +""" +function pycolumntable(::Type{T}, src) where {T} + cols = Tables.columns(src) + pydict(T, pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) +end +pycolumntable(::Type{T}; cols...) where {T} = pycolumntable(T, cols) +pycolumntable(src) = pycolumntable(PyObject, src) +pycolumntable(; opts...) = pycolumntable(PyObject, opts) +export pycolumntable + +""" + pyrowtable([T=PyObject,] src) :: T + +Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. +""" +function pyrowtable(::Type{T}, src) where {T} + rows = Tables.rows(src) + names = Tables.columnnames(rows) + pynames = [pystr(String(n)) for n in names] + pylist(T, pydict(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) +end +pyrowtable(::Type{T}; cols...) where {T} = pyrowtable(T, cols) +pyrowtable(src) = pyrowtable(PyObject, src) +pyrowtable(; opts...) = pyrowtable(PyObject, opts) +export pyrowtable + +# """ +# pypandasdataframe([T=PyObject,] [src]; ...) :: T + +# Construct a pandas dataframe from `src`. + +# Usually equivalent to `pyimport("pandas").DataFrame(src, ...)`, but `src` may also be `Tables.jl`-compatible table. +# """ +# pypandasdataframe(t::PyObject; opts...) = pypandasdataframetype(t; opts...) +# pypandasdataframe(; opts...) = pypandasdataframetype(; opts...) +# function pypandasdataframe(t; opts...) +# if Tables.istable(t) +# cs = Tables.columns(t) +# pypandasdataframetype(pydict_fromstringiter(string(c) => asvector(Tables.getcolumn(cs, c)) for c in Tables.columnnames(cs)); opts...) +# else +# pypandasdataframetype(t; opts...) +# end +# end +# export pypandasdataframe + +multidict(src) = Dict(k=>v for (ks,v) in src for k in (ks isa Vector ? ks : [ks])) + +""" + PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) + +Wrap the Pandas dataframe `o` as a Julia table. + +This object satisfies the `Tables.jl` and `TableTraits.jl` interfaces. + +- `:indexname` is the name of the index column when converting this to a table, and may be `nothing` to exclude the index. +- `:columntypes` is an iterable of `columnname=>type` or `[columnnames...]=>type` pairs, used when converting to a table. +- `:copy` is true to copy columns on conversion. +""" +struct PyPandasDataFrame + ref :: PyRef + indexname :: Union{Symbol, Nothing} + columntypes :: Dict{Symbol, Type} + copy :: Bool +end +PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) = PyPandasDataFrame(PyRef(o), indexname, multidict(columntypes), copy) +export PyPandasDataFrame + +ispyreftype(::Type{PyPandasDataFrame}) = true +pyptr(df::PyPandasDataFrame) = df.ref +Base.unsafe_convert(::Type{CPyPtr}, df::PyPandasDataFrame) = checknull(pyptr(df)) +C.PyObject_TryConvert__initial(o, ::Type{PyPandasDataFrame}) = C.putresult(PyPandasDataFrame(pyborrowedref(o))) + +Base.show(io::IO, x::PyPandasDataFrame) = print(io, pystr(String, x)) +Base.show(io::IO, mime::_py_mimetype, o::PyPandasDataFrame) = _py_mime_show(io, mime, o) +Base.showable(mime::_py_mimetype, o::PyPandasDataFrame) = _py_mime_showable(mime, o) + +### Tables.jl / TableTraits.jl integration + +Tables.istable(::Type{PyPandasDataFrame}) = true +Tables.columnaccess(::Type{PyPandasDataFrame}) = true +function Tables.columns(x::PyPandasDataFrame) + error("not implemented") + # # columns + # names = Symbol[Symbol(pystr(String, c)) for c in x.o.columns] + # columns = PyVector[PyVector{get(x.columntypes, c, missing)}(x.o[pystr(c)].to_numpy()) for c in names] + # # index + # if x.indexname !== nothing + # if x.indexname ∈ names + # error("table already has column called $(x.indexname), cannot use it for index") + # else + # pushfirst!(names, x.indexname) + # pushfirst!(columns, PyArray{get(x.columntypes, x.indexname, missing)}(x.o.index.to_numpy())) + # end + # end + # if x.copy + # columns = map(copy, columns) + # end + # return NamedTuple{Tuple(names)}(Tuple(columns)) +end + +IteratorInterfaceExtensions.isiterable(x::PyPandasDataFrame) = true +IteratorInterfaceExtensions.getiterator(x::PyPandasDataFrame) = IteratorInterfaceExtensions.getiterator(Tables.rows(x)) + +TableTraits.isiterabletable(x::PyPandasDataFrame) = true diff --git a/src/PyRef.jl b/src/PyRef.jl new file mode 100644 index 00000000..4faf219d --- /dev/null +++ b/src/PyRef.jl @@ -0,0 +1,66 @@ +""" + PyRef([x]) + +A reference to a Python object converted from `x`, or a null reference if `x` is not given. + +This is baically just a mutable reference to a pointer to a Python object. +It owns the reference (if non-NULL) and automatically decrefs it when finalized. + +Building block for more complex wrapper types such as `PyObject` and `PyList`. +""" +mutable struct PyRef + ptr :: CPyPtr + PyRef(::Val{:new}, ptr::Ptr, borrowed::Bool) = begin + x = new(CPyPtr(ptr)) + borrowed && C.Py_IncRef(ptr) + finalizer(x) do x + if CONFIG.isinitialized + ptr = x.ptr + if !isnull(ptr) + with_gil(false) do + C.Py_DecRef(ptr) + end + x.ptr = CPyPtr() + end + end + end + x + end +end +export PyRef + +pynewref(x::Ptr, check::Bool=false) = (check && isnull(x)) ? pythrow() : PyRef(Val(:new), x, false) +pyborrowedref(x::Ptr, check::Bool=false) = (check && isnull(x)) ? pythrow() : PyRef(Val(:new), x, true) + +ispyreftype(::Type{PyRef}) = true +pyptr(x::PyRef) = x.ptr +isnull(x::PyRef) = isnull(pyptr(x)) +Base.unsafe_convert(::Type{CPyPtr}, x::PyRef) = pyptr(x) +C.PyObject_TryConvert__initial(o, ::Type{PyRef}) = C.putresult(pyborrowedref(o)) + +PyRef(x) = begin + ptr = C.PyObject_From(x) + isnull(ptr) && pythrow() + pynewref(ptr) +end +PyRef() = pynewref(CPyPtr()) + +Base.convert(::Type{PyRef}, x::PyRef) = x +Base.convert(::Type{PyRef}, x) = PyRef(x) + +# Cache some common standard modules +for name in ["os", "io", "sys", "pprint", "collections", "collections.abc", "numbers", "fractions", "datetime", "numpy", "pandas"] + f = Symbol("py", replace(name, "."=>""), "module") + rf = Symbol("_", f) + @eval $rf = PyRef() + @eval $f(::Type{T}) where {T} = begin + r = $rf + if isnull(r.ptr) + m = pyimport(PyRef, $name) + C.Py_IncRef(m.ptr) + r.ptr = m.ptr + end + (r isa T) ? r : pyconvert(T, r) + end + @eval $f() = $f(PyObject) +end diff --git a/src/PySet.jl b/src/PySet.jl index 8aa10122..bbabc99d 100644 --- a/src/PySet.jl +++ b/src/PySet.jl @@ -1,73 +1,83 @@ """ - PySet{T=PyObject}(o=pyset()) + PySet{T=PyObject}([o]) Wrap the Python set `o` (or anything satisfying the set interface) as a Julia set with elements of type `T`. + +If `o` is not given, an empty set is created. """ struct PySet{T} <: AbstractSet{T} - o :: PyObject - PySet{T}(o::PyObject) where {T} = new{T}(o) + ref :: PyRef + PySet{T}(o) where {T} = new{T}(PyRef(o)) + PySet{T}() where {T} = new{T}(PyRef()) end -PySet{T}(o=pyset()) where {T} = PySet{T}(pyset(o)) -PySet(o=pyset()) = PySet{PyObject}(o) +PySet(o) = PySet{PyObject}(o) +PySet() = PySet{PyObject}() export PySet -pyobject(x::PySet) = x.o +ispyreftype(::Type{<:PySet}) = true +pyptr(x::PySet) = begin + ptr = x.ref.ptr + if isnull(ptr) + ptr = x.ref.ptr = C.PySet_New(C_NULL) + end + ptr +end +Base.unsafe_convert(::Type{CPyPtr}, x::PySet) = checknull(pyptr(x)) +C.PyObject_TryConvert__initial(o, ::Type{T}) where {T<:PySet} = C.putresult(T(pyborrowedref(o))) -function Base.iterate(x::PySet{T}, it=pyiter(x.o)) where {T} +Base.iterate(x::PySet{T}, it::PyRef) where {T} = begin ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing + if ptr != C_NULL + r = C.PyObject_Convert(ptr, T) + C.Py_DecRef(ptr) + checkm1(r) + (C.takeresult(T), it) + elseif C.PyErr_IsSet() + pythrow() else - (pyconvert(T, pynewobject(ptr)), it) + nothing end end +Base.iterate(x::PySet) = begin + it = C.PyObject_GetIter(x) + isnull(it) && pythrow() + iterate(x, pynewref(it)) +end -Base.length(x::PySet) = Int(pylen(x.o)) +# Base.length(x::PySet) = @pyv `len($x)`::Int +Base.length(x::PySet) = Int(checkm1(C.PyObject_Length(x))) -function Base.in(_v, x::PySet{T}) where {T} - v = tryconvert(T, _v) - v === PyConvertFail() ? false : pycontains(x, v) +Base.in(_v, x::PySet{T}) where {T} = begin + v = tryconvertref(T, _v) + v === PYERR() && pythrow() + v === NOTIMPLEMENTED() && return false + pycontains(x, v) end -function Base.push!(x::PySet{T}, v) where {T} - x.o.add(convert(T, v)) - x -end +Base.push!(x::PySet{T}, v) where {T} = (@py `$x.add($(convertref(T, v)))`; x) -function Base.delete!(x::PySet{T}, _v) where {T} - v = tryconvert(T, _v) - v === PyConvertFail() || x.o.discard(v) +Base.delete!(x::PySet{T}, _v) where {T} = begin + v = tryconvertref(T, _v) + v === PYERR() && pythrow() + v === NOTIMPLEMENTED() && return x + @py `$x.discard($v)` x end -function Base.pop!(x::PySet{T}) where {T} - v = x.o.pop() - pyconvert(T, v) -end - -function Base.pop!(x::PySet{T}, _v) where {T} - v = convert(T, _v) - x.o.remove(v) - v -end +Base.pop!(x::PySet{T}) where {T} = @pyv `$x.pop()`::T -function Base.pop!(x::PySet{T}, _v, d) where {T} +Base.pop!(x::PySet{T}, _v) where {T} = begin v = tryconvert(T, _v) - (v !== PyConvertFail() && v in x) ? pop!(x, v) : d + v === PYERR() && pythrow() + (v !== NOTIMPLEMENTED() && v in x) ? (delete!(x, v); v) : error("not an element") end -function Base.filter!(f, x::PySet{T}) where {T} - d = pylist() - for _v in x.o - if !f(pyconvert(T, _v)) - d.append(_v) - end - end - x.o.difference_update(d) - x +Base.pop!(x::PySet{T}, _v, d) where {T} = begin + v = tryconvert(T, _v) + v === PYERR() && pythrow() + (v !== NOTIMPLEMENTED() && v in x) ? (delete!(x, v); v) : d end -Base.empty!(x::PySet) = (x.o.clear(); x) +Base.empty!(x::PySet) = (@py `$x.clear()`; x) -Base.copy(x::PySet) = typeof(x)(x.o.copy()) +Base.copy(x::PySet) = @pyv `$x.copy()`::typeof(x) diff --git a/src/Python.jl b/src/Python.jl index c0c22e8e..64b22d6b 100644 --- a/src/Python.jl +++ b/src/Python.jl @@ -52,86 +52,56 @@ Base.show(io::IO, ::MIME"text/plain", c::Config) = end const CONFIG = Config() -# C API -include("cpython.jl") - -const C = CPython -const CPyPtr = C.PyPtr -struct CPyObjRef - ptr :: CPyPtr -end - -# core -include("object.jl") -include("error.jl") -include("import.jl") -include("gil.jl") +""" + ispyreftype(::Type{T}) -# abstract interfaces -include("number.jl") -include("sequence.jl") +True if `T` is a wrapper type for a single Python reference. -# fundamental objects -include("type.jl") -include("none.jl") +Such objects must implement: +- `pyptr(o::T) =` the underlying pointer (or NULL on error) +- `unsafe_convert(::Type{CPyPtr}, o::T) = checknull(pyptr(o))` +""" +ispyreftype(::Type) = false -# numeric objects -include("bool.jl") -include("int.jl") -include("float.jl") -include("complex.jl") +""" + pyptr(o) -# sequence objects -include("str.jl") -include("bytes.jl") -include("bytearray.jl") -include("tuple.jl") -include("list.jl") +Retrieve the underlying Python object pointer from o. +""" +function pyptr end -# mapping objects -include("dict.jl") -include("set.jl") +convertref(::Type{T}, x) where {T} = ispyreftype(T) ? x : convert(T, x) +tryconvertref(::Type{T}, x) where {T} = ispyreftype(T) ? x : tryconvert(T, x) -# function objects -include("function.jl") +# C API +include("cpython/CPython.jl") -# other objects -include("slice.jl") -include("range.jl") +const C = CPython +const CPyPtr = C.PyPtr -# standard library -include("builtins.jl") +include("gil.jl") include("eval.jl") -include("stdlib.jl") -include("io.jl") -include("fraction.jl") -include("datetime.jl") -include("collections.jl") - -# other packages -include("pandas.jl") -include("numpy.jl") -include("matplotlib.jl") +include("builtins.jl") -# other Julia wrappers around Python values -include("PyIterable.jl") -include("PyList.jl") +include("PyRef.jl") +include("PyCode.jl") +include("PyInternedString.jl") +include("PyException.jl") +include("PyObject.jl") include("PyDict.jl") +include("PyList.jl") include("PySet.jl") -include("PyObjectArray.jl") +include("PyIterable.jl") +include("PyIO.jl") include("PyBuffer.jl") include("PyArray.jl") -include("PyIO.jl") +include("PyObjectArray.jl") +include("PyPandasDataFrame.jl") -# other functionality -include("convert.jl") -include("newtype.jl") include("julia.jl") -include("base.jl") -include("pywith.jl") include("gui.jl") +include("matplotlib.jl") -# initialize include("init.jl") end # module diff --git a/src/base.jl b/src/base.jl deleted file mode 100644 index 54fc70ea..00000000 --- a/src/base.jl +++ /dev/null @@ -1,325 +0,0 @@ -function Base.show(io::IO, o::PyObject) - s = pyrepr(String, o) - if get(io, :typeinfo, Any) == typeof(o) - print(io, s) - elseif startswith(s, "<") && endswith(s, ">") - print(io, "") - end -end - -Base.print(io::IO, o::PyObject) = print(io, pystr(String, o)) - -function Base.show(io::IO, ::MIME"text/plain", o::PyObject) - h, w = displaysize(io) - h -= 3 - x = try - pystr_asjuliastring(pypprintmodule.pformat(o, width=w)) - catch - pyrepr(String, o) - end - if get(io, :limit, true) - if '\n' ∈ x - # multiple lines - # each one is truncated to one screen width - print(io, "py:") - h -= 1 - xs = split(x, '\n') - printlines(xs) = - for x in xs - if length(x) ≤ w - print(io, '\n', x) - else - print(io, '\n', x[1:nextind(x, 0, w-1)], '…') - end - end - if length(xs) ≤ h - # all lines fit on screen - printlines(xs) - else - # too many lines, skip the middle ones - h -= 1 - h2 = cld(h, 2) - h3 = (length(xs)+1)-(h-h2) - printlines(xs[1:h2]) - print(io, "\n ... [skipping $(h3-h2-1) lines] ...") - printlines(xs[h3:end]) - end - elseif length(x) ≤ w-4 - # one short line - print(io, "py: ", x) - return - else - # one long line - println(io, "py:") - h -= 1 - a = h * w - if length(x) ≤ a - # whole string fits on screen - print(io, x) - else - # too long, skip the middle - h -= 1 - h2 = cld(h, 2) - i2 = nextind(x, 0, h2 * w) - i3 = prevind(x, ncodeunits(x)+1, (h-h2)*w) - println(io, x[1:i2]) - println(io, " ... [skipping $(length(x[nextind(x,i2):prevind(x,i3)])) characters] ...") - print(io, x[i3:end]) - end - end - else - print(io, "py: ", x) - end -end - -Base.hasproperty(o::PyObject, k::Union{Symbol,AbstractString}) = pyhasattr(o, k) - -Base.getproperty(o::PyObject, k::Symbol) = - if k == :jl! - (T=Any) -> pyconvert(T, o) - elseif k == :jl!i - pyconvert(Int, o) - elseif k == :jl!u - pyconvert(UInt, o) - elseif k == :jl!b - pytruth(o) - elseif k == :jl!s - pystr(String, o) - elseif k == :jl!r - pyrepr(String, o) - elseif k == :jl!f - pyconvert(Cdouble, o) - elseif k == :jl!c - pyconvert(Complex{Cdouble}, o) - elseif k == :jl!list - (args...) -> PyList{args...}(o) - elseif k == :jl!dict - (args...) -> PyDict{args...}(o) - elseif k == :jl!set - (args...) -> PySet{args...}(o) - elseif k == :jl!buffer - () -> PyBuffer(o) - elseif k == :jl!array - (args...) -> PyArray{args...}(o) - elseif k == :jl!vector - (args...) -> PyVector{args...}(o) - elseif k == :jl!matrix - (args...) -> PyMatrix{args...}(o) - elseif k == :jl!pandasdf - (; opts...) -> PyPandasDataFrame(o; opts...) - elseif k == :jl!io - (; opts...) -> PyIO(o; opts...) - else - pygetattr(o, k) - end -Base.getproperty(o::PyObject, k::AbstractString) = - startswith(k, "jl!") ? getproperty(o, Symbol(k)) : pygetattr(o, k) - -Base.setproperty!(o::PyObject, k::Symbol, v) = (pysetattr(o, k, v); o) -Base.setproperty!(o::PyObject, k::AbstractString, v) = (pysetattr(o, k, v); o) - -function Base.propertynames(o::PyObject) - # this follows the logic of rlcompleter.py - function classmembers(c) - r = pydir(c) - if pyhasattr(c, "__bases__") - for b in c.__bases__ - r = pyiadd(r, classmembers(b)) - end - end - return r - end - words = pyset(pydir(o)) - words.discard("__builtins__") - if pyhasattr(o, "__class__") - words.add("__class__") - words.update(classmembers(o.__class__)) - end - r = [Symbol(pystr(String, x)) for x in words] - return r -end - - -Base.getindex(o::PyObject, k) = pygetitem(o, k) -Base.getindex(o::PyObject, k...) = pygetitem(o, k) - -Base.setindex!(o::PyObject, v, k) = (pysetitem(o, k, v); o) -Base.setindex!(o::PyObject, v, k...) = (pysetitem(o, k, v); o) - -Base.delete!(o::PyObject, k) = (pydelitem(o, k); o) - -Base.length(o::PyObject) = Int(pylen(o)) - -Base.in(v, o::PyObject) = pycontains(o, v) - -(f::PyObject)(args...; kwargs...) = pycall(f, args...; kwargs...) - -Base.eltype(::Type{T}) where {T<:PyObject} = PyObject -Base.IteratorSize(::Type{T}) where {T<:PyObject} = Base.SizeUnknown() -function Base.iterate(o::PyObject, it=pyiter(o)) - ptr = C.PyIter_Next(it) - if ptr == C_NULL - pyerrcheck() - nothing - else - (pynewobject(ptr), it) - end -end - -# comparisons -Base.:(==)(o1::PyObject, o2::PyObject) = pyeq(Bool, o1, o2) -Base.:(!=)(o1::PyObject, o2::PyObject) = pyne(Bool, o1, o2) -Base.:(< )(o1::PyObject, o2::PyObject) = pylt(Bool, o1, o2) -Base.:(<=)(o1::PyObject, o2::PyObject) = pyle(Bool, o1, o2) -Base.:(> )(o1::PyObject, o2::PyObject) = pygt(Bool, o1, o2) -Base.:(>=)(o1::PyObject, o2::PyObject) = pyge(Bool, o1, o2) -Base.isequal(o1::PyObject, o2::PyObject) = pyeq(Bool, o1, o2) -Base.isless(o1::PyObject, o2::PyObject) = pylt(Bool, o1, o2) - -# unary arithmetic -Base.:(-)(o::PyObject) = pyneg(o) -Base.:(+)(o::PyObject) = pypos(o) -Base.abs(o::PyObject) = pyabs(o) -Base.:(~)(o::PyObject) = pyinv(o) - -# binary arithmetic -Base.:(+)(o1::PyObject, o2::PyObject) = pyadd(o1, o2) -Base.:(-)(o1::PyObject, o2::PyObject) = pysub(o1, o2) -Base.:(*)(o1::PyObject, o2::PyObject) = pymul(o1, o2) -Base.:(/)(o1::PyObject, o2::PyObject) = pytruediv(o1, o2) -Base.:fld(o1::PyObject, o2::PyObject) = pyfloordiv(o1, o2) -Base.:mod(o1::PyObject, o2::PyObject) = pymod(o1, o2) -Base.:(^)(o1::PyObject, o2::PyObject) = pypow(o1, o2) -Base.:(<<)(o1::PyObject, o2::PyObject) = pylshift(o1, o2) -Base.:(>>)(o1::PyObject, o2::PyObject) = pyrshift(o1, o2) -Base.:(&)(o1::PyObject, o2::PyObject) = pyand(o1, o2) -Base.:xor(o1::PyObject, o2::PyObject) = pyxor(o1, o2) -Base.:(|)(o1::PyObject, o2::PyObject) = pyor(o1, o2) - -Base.:(+)(o1::PyObject, o2::Number) = pyadd(o1, o2) -Base.:(-)(o1::PyObject, o2::Number) = pysub(o1, o2) -Base.:(*)(o1::PyObject, o2::Number) = pymul(o1, o2) -Base.:(/)(o1::PyObject, o2::Number) = pytruediv(o1, o2) -Base.:fld(o1::PyObject, o2::Number) = pyfloordiv(o1, o2) -Base.:mod(o1::PyObject, o2::Number) = pymod(o1, o2) -Base.:(^)(o1::PyObject, o2::Number) = pypow(o1, o2) -Base.:(<<)(o1::PyObject, o2::Number) = pylshift(o1, o2) -Base.:(>>)(o1::PyObject, o2::Number) = pyrshift(o1, o2) -Base.:(&)(o1::PyObject, o2::Number) = pyand(o1, o2) -Base.:xor(o1::PyObject, o2::Number) = pyxor(o1, o2) -Base.:(|)(o1::PyObject, o2::Number) = pyor(o1, o2) - -Base.:(+)(o1::Number, o2::PyObject) = pyadd(o1, o2) # Defining +(::Any, ::PyObject) like this hangs Julia v1.5.2-v1.5.3 (at least) during precompilation -Base.:(-)(o1::Number, o2::PyObject) = pysub(o1, o2) -Base.:(*)(o1::Number, o2::PyObject) = pymul(o1, o2) -Base.:(/)(o1::Number, o2::PyObject) = pytruediv(o1, o2) -Base.:fld(o1::Number, o2::PyObject) = pyfloordiv(o1, o2) -Base.:mod(o1::Number, o2::PyObject) = pymod(o1, o2) -# Base.:(^)(o1::Number, o2::PyObject) = pypow(o1, o2) -# Base.:(<<)(o1::Number, o2::PyObject) = pylshift(o1, o2) -# Base.:(>>)(o1::Number, o2::PyObject) = pyrshift(o1, o2) -Base.:(&)(o1::Number, o2::PyObject) = pyand(o1, o2) -Base.:xor(o1::Number, o2::PyObject) = pyxor(o1, o2) -Base.:(|)(o1::Number, o2::PyObject) = pyor(o1, o2) - -# ternary arithmetic -Base.powermod(o1::PyObject, o2::Union{PyObject,Number}, o3::Union{PyObject,Number}) = pypow(o1, o2, o3) - -Base.zero(::Type{PyObject}) = pyint(0) -Base.one(::Type{PyObject}) = pyint(1) - -function Base.Docs.getdoc(o::PyObject) - function tryget(f, g=identity) - a = try - f() - catch - return nothing - end - pyisnone(a) ? nothing : g(a) - end - # function name(o) - # n = tryget(()->o.__name__) - # n === nothing && return nothing - # m = tryget(()->o.__module__) - # (m === nothing || string(m) == "builtins") ? string(n) : string(m, ".", n) - # end - getname(o) = tryget(()->o.__name__, string) - getdoc(o) = tryget(()->o.__doc__, string ∘ ins.cleandoc) - - docs = [] - - # Short description - ins = pyimport("inspect") - desc, name = if ins.ismodule(o).jl!b - (pyhasattr(o, "__path__") ? "package" : "module"), gname(o) - elseif ins.isgetsetdescriptor(o).jl!b - ("getset descriptor", "$(getname(o.__objclass__)).$(o.__name__)") - elseif ins.ismemberdescriptor(o).jl!b - ("member descriptor", "$(getname(o.__objclass__)).$(o.__name__)") - elseif ins.isclass(o).jl!b - ("class", getname(o)) - elseif ins.isfunction(o).jl!b || ins.isbuiltin(o).jl!b - ("function", getname(o)) - elseif ins.ismethod(o).jl!b - ("method", getname(o)) - else - ("object of type", getname(pytype(o))) - end - push!(docs, Markdown.Paragraph(["Python ", desc, " ", Markdown.Code(name), "."])) - - if ins.isroutine(o).jl!b || ins.isclass(o).jl!b - try - push!(docs, Markdown.Code("python", "$(o.__name__)$(ins.signature(o))")) - catch - end - end - - # Maybe document the class instead - doc = getdoc(o) - if doc === nothing && !ins.ismodule(o).jl!b && !ins.isclass(o).jl!b && !ins.isroutine(o).jl!b && !ins.isdatadescriptor(o).jl!b - o = pyhasattr(o, "__origin__") ? o.__origin__ : pytype(o) - doc = getdoc(o) - end - doc === nothing || push!(docs, Markdown.Paragraph([Markdown.Text(doc)])) - - # Done - Markdown.MD(docs) -end -Base.Docs.Binding(o::PyObject, k::Symbol) = getproperty(o, k) - -for (mime, method) in ((MIME"text/html", "_repr_html_"), - (MIME"text/markdown", "_repr_markdown_"), - (MIME"text/json", "_repr_json_"), - (MIME"application/javascript", "_repr_javascript_"), - (MIME"application/pdf", "_repr_pdf_"), - (MIME"image/jpeg", "_repr_jpeg_"), - (MIME"image/png", "_repr_png_"), - (MIME"image/svg+xml", "_repr_svg_"), - (MIME"text/latex", "_repr_latex_")) - T = istextmime(mime()) ? String : Vector{UInt8} - @eval begin - function Base.show(io::IO, mime::$mime, o::PyObject) - try - x = pygetattr(o, $method)() - pyisnone(x) || return write(io, pyconvert($T, x)) - catch - end - throw(MethodError(show, (io, mime, o))) - end - function Base.showable(::$mime, o::PyObject) - try - x = pygetattr(o, $method)() - if pyisnone(x) - false - else - pyconvert($T, x) - true - end - catch - false - end - end - end -end diff --git a/src/bool.jl b/src/bool.jl deleted file mode 100644 index 5d722780..00000000 --- a/src/bool.jl +++ /dev/null @@ -1,22 +0,0 @@ -const pybooltype = pylazyobject(() -> pybuiltins.bool) -export pybooltype - -const pytrue = pylazyobject(() -> pybuiltins.True) -const pyfalse = pylazyobject(() -> pybuiltins.False) -export pytrue, pyfalse - -pybool(args...; opts...) = pybooltype(args...; opts...) -pybool(o::Bool) = o ? pytrue : pyfalse -export pybool - -pyisbool(o::PyObject) = pytypecheck(o, pybooltype) -export pyisbool - -function pybool_tryconvert(::Type{T}, o::PyObject) where {T} - x = pytruth(o) - if Bool <: T - pytruth(o) - else - tryconvert(T, x) - end -end diff --git a/src/builtins.jl b/src/builtins.jl index dea7aba1..bd77e524 100644 --- a/src/builtins.jl +++ b/src/builtins.jl @@ -1,60 +1,816 @@ -const pybuiltins = pylazyobject(() -> pyimport("builtins")) -export pybuiltins - -# types -for p in [:classmethod, :enumerate, :filter, :map, :property, :reversed, :staticmethod, :super, :zip] - j = Symbol(:py, p, :type) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - @eval export $j +cpyop(f::Function, x) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + r = f(xo) + C.Py_DecRef(xo) + r end -# functions -for p in [:all, :any, :chr, :compile, :eval, :exec, :format, :help, :hex, :id, :max, :min, :next, :oct, :open, :ord, :print, :round, :sorted, :sum, :vars] - j = Symbol(:py, p, :func) - jf = Symbol(:py, p) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - if p in [:help, :print, :exec] - @eval $jf(args...; opts...) = ($j(args...; opts...); nothing) +cpyop(f::Function, x, y) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + yo = C.PyObject_From(y) + isnull(yo) && (C.Py_DecRef(xo); pythrow()) + r = f(xo, yo) + C.Py_DecRef(xo) + C.Py_DecRef(yo) + r +end + +cpyop(f::Function, x, y, z) = begin + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + yo = C.PyObject_From(y) + isnull(yo) && (C.Py_DecRef(xo); pythrow()) + zo = C.PyObject_From(z) + isnull(yo) && (C.Py_DecRef(xo); C.Py_DecRef(yo); pythrow()) + r = f(xo, yo, zo) + C.Py_DecRef(xo) + C.Py_DecRef(yo) + C.Py_DecRef(zo) + r +end + +cpyop(::Type{T}, f::Function, args...) where {T} = checknullconvert(T, cpyop(f, args...)) + +""" + pyconvert(T, x) :: T + +Convert Python object `x` to a `T`. +""" +pyconvert(::Type{T}, x) where {T} = checknullconvert(T, C.PyObject_From(x)) +export pyconvert + +""" + pyis(x, y) :: Bool + +Equivalent to `x is y` in Python. +""" +pyis(x::X, y::Y) where {X,Y} = begin + if ispyreftype(X) + xo = pyptr(x) + isnull(xo) && pythrow() else - @eval $jf(args...; opts...) = $j(args...; opts...) + xo = C.PyObject_From(x) + isnull(xo) && pythrow() + C.Py_DecRef(xo) end - @eval export $jf + if ispyreftype(Y) + yo = pyptr(y) + isnull(yo) && pythrow() + else + yo = C.PyObject_From(y) + isnull(yo) && pythrow() + C.Py_DecRef(yo) + end + xo == yo end +export pyis + +""" + pyhasattr(x, k) :: Bool + +Equivalent to `hasattr(x, k)` in Python, returned as a `Bool`. +""" +pyhasattr(x, k) = checkm1(cpyop(C.PyObject_HasAttr, x, k)) != 0 +pyhasattr(x, k::String) = checkm1(cpyop(xo -> C.PyObject_HasAttrString(xo, k), x)) != 0 +pyhasattr(x, k::Symbol) = pyhasattr(x, string(k)) +export pyhasattr + +""" + pygetattr([T=PyObject,] x, k) :: T + +Equivalent to `x.k` or `getattr(x, k)` in Python. +""" +pygetattr(::Type{T}, x, k) where {T} = cpyop(T, C.PyObject_GetAttr, x, k) +pygetattr(::Type{T}, x, k::String) where {T} = cpyop(T, xo->C.PyObject_GetAttrString(xo, k), x) +pygetattr(::Type{T}, x, k::Symbol) where {T} = pygetattr(T, x, string(k)) +pygetattr(x, k) = pygetattr(PyObject, x, k) +export pygetattr + +""" + pysetattr(x, k, v) + +Equivalent to `x.k = v` or `setattr(x, k, v)` in Python, but returns `x`. +""" +pysetattr(x, k, v) = (checkm1(cpyop(C.PyObject_SetAttr, x, k, v)); x) +pysetattr(x, k::String, v) = (checkm1(cpyop((xo, vo) -> C.PyObject_SetAttrString(xo, k, vo), x, v)); x) +pysetattr(x, k::Symbol, v) = pysetattr(x, string(k), v) +export pysetattr + +""" + pydir([T=PyObject,] x) :: T + +Equivalent to `dir(x)` in Python. +""" +pydir(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Dir, x) +pydir(x) = pydir(PyObject, x) +export pydir + +""" + pycall([T=PyObject,] f, args...; kwargs...) :: T + +Equivalent to `f(*args, **kwargs)` in Python. +""" +pycall(::Type{T}, f, args...; opts...) where {T} = cpyop(T, fo -> C.PyObject_CallArgs(fo, args, opts), f) +pycall(f, args...; opts...) = pycall(PyObject, f, args...; opts...) +export pycall + +""" + pyrepr([T=PyObject,] x) :: T + +Equivalent to `repr(x)` in Python. +""" +pyrepr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Repr, x) +pyrepr(x) = pyrepr(PyObject, x) +export pyrepr + +""" + pystr([T=PyObject,] x) :: T + +Equivalent to `str(x)` in Python. +""" +pystr(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Str, x) +pystr(::Type{T}, x::Union{String, SubString{String}, Vector{Int8}, Vector{UInt8}}) where {T} = checknullconvert(T, C.PyUnicode_From(x)) +pystr(x) = pystr(PyObject, x) +export pystr + +""" + pybytes([T=PyObject,] x) :: T + +Equivalent to `str(x)` in Python. +""" +pybytes(::Type{T}, x) where {T} = cpyop(T, C.PyObject_Bytes, x) +pybytes(::Type{T}, x::Union{Vector{Int8}, Vector{UInt8}, String, SubString{String}}) where {T} = checknullconvert(T, C.PyBytes_From(x)) +pybytes(x) = pybytes(PyObject, x) +export pybytes + +""" + pylen(x) :: Integer + +Equivalent to `len(x)` in Python. +""" +pylen(x) = checkm1(cpyop(C.PyObject_Length, x)) + +""" + pycontains(x, v) :: Bool + +Equivalent to `v in x` in Python. +""" +pycontains(x, v) = checkm1(cpyop(C.PySequence_Contains, x, v)) != 0 +export pycontains + +""" + pyin(v, x) :: Bool + +Equivalent to `v in x` in Python. +""" +pyin(v, x) = pycontains(x, v) +export pyin + +""" + pygetitem([T=PyObject,] x, k) :: T + +Equivalent to `x[k]` or `getitem(x, k)` in Python. +""" +pygetitem(::Type{T}, x, k) where {T} = cpyop(T, C.PyObject_GetItem, x, k) +pygetitem(x, k) = pygetitem(PyObject, x, k) +export pygetitem + +""" + pysetitem(x, k, v) + +Equivalent to `x[k] = v` or `setitem(x, k, v)` in Python, but returns `x`. +""" +pysetitem(x, k, v) = (checkm1(cpyop(C.PyObject_SetItem, x, k, v)); x) +export pysetitem + +""" + pydelitem(x, k) + +Equivalent to `del x[k]` or `delitem(x, k)` in Python, but returns x. +""" +pydelitem(x, k) = (checkm1(cpyop(C.PyObject_DelItem, x, k)); x) +export pydelitem + +""" + pynone([T=PyObject]) :: T + +Equivalent to `None` in Python. +""" +pynone(::Type{T}) where {T} = (checkm1(C.PyObject_Convert(C.Py_None(), T)); takeresult(T)) +pynone() = pynone(PyObject) +export pynone + +""" + pybool([T=PyObject,] ...) :: T -# singletons -for p in [:Ellipsis, :NotImplemented] - j = Symbol(:py, lowercase(string(p))) - @eval const $j = pylazyobject(() -> pybuiltins.$p) - @eval export $j +Equivalent to `bool(...)` in Python. +""" +pybool(::Type{T}, x::Bool) where {T} = checknullconvert(T, C.PyBool_From(x)) +pybool(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyBool_Type(), args, kwargs)) +pybool(args...; kwargs...) = pybool(PyObject, args...; kwargs...) +export pybool + +""" + pyint([T=PyObject,] ...) :: T + +Equivalent to `int(...)` in Python. +""" +pyint(::Type{T}, x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) where {T} = + checknullconvert(T, C.PyLong_From(x)) +pyint(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Long, x) +pyint(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyLong_Type(), args, kwargs)) +pyint(args...; kwargs...) = pyint(PyObject, args...; kwargs...) +export pyint + +""" + pyfloat([T=PyObject,] ...) :: T + +Equivalent to `float(...)` in Python. +""" +pyfloat(::Type{T}, x::Union{Float16,Float32,Float64}) where {T} = checknullconvert(T, C.PyFloat_From(x)) +pyfloat(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Float, x) +pyfloat(::Type{T}, args...; kwargs...) where {T} = checknullconvert(T, C.PyObject_CallArgs(C.PyFloat_Type(), args, kwargs)) +pyfloat(args...; kwargs...) = pyfloat(PyObject, args...; kwargs...) +export pyfloat + +""" + pyimport([T=PyObject,] name) :: T + pyimport([T=PyObject,] name=>attr) :: T + pyimport([T=PyObject,] name=>(attr,...)) :: Tuple{T,...} + +Imports and returns the Python module `name`. + +If additionally `attr` is given, the given attribute of the module is returned instead. +It may also be a tuple of attributes. + +If several arguments are given, each one is imported and a tuple is returned. +""" +pyimport(::Type{T}, m) where {T} = cpyop(T, C.PyImport_Import, m) +pyimport(::Type{T}, m::String) where {T} = checknullconvert(T, C.PyImport_ImportModule(m)) +pyimport(::Type{T}, m::Pair) where {T} = begin + r = pyimport(PyObject, m[1]) + m[2] isa Tuple ? map(k->pygetattr(T, r, k), m[2]) : pygetattr(T, r, m[2]) end +pyimport(::Type{T}, m1, m2, ms...) where {T} = map(m->pyimport(T, m), (m1, m2, ms...)) +pyimport(m1, ms...) = pyimport(PyObject, m1, ms...) +export pyimport + +""" + pytruth(x) :: Bool + +The truthyness of `x`, equivalent to `bool(x)` or `not not x` in Python, or to `pybool(Bool, x)`. +""" +pytruth(x) = checkm1(cpyop(C.PyObject_IsTrue, x)) != 0 +export pytruth + +""" + pyissubclass(x, y) :: Bool + +Equivalent to `issubclass(x, y)` in Python. +""" +pyissubclass(x, y) = checkm1(cpyop(C.PyObject_IsSubclass, x, y)) != 0 +export pyissubclass + +""" + pyisinstance(x, y) :: Bool + +Equivalent to `isinstance(x, y)` in Python. +""" +pyisinstance(x, y) = checkm1(cpyop(C.PyObject_IsInstance, x, y)) != 0 +export pyisinstance + +""" + pyhash(x) :: Integer + +Equivalent to `hash(x)` in Python. +""" +pyhash(x) = checkm1(cpyop(C.PyObject_Hash, x)) +export pyhash + +""" + pycompare([T=PyObject,] x, op, y) :: T + +Equivalent to `x op y` in Python where `op` is one of: `=`, `≠`, `<`, `≤`, `>`, `≥`. +""" +pycompare(::Type{T}, x, ::typeof(==), y) where {T} = _pycompare(T, x, C.Py_EQ, y) +pycompare(::Type{T}, x, ::typeof(!=), y) where {T} = _pycompare(T, x, C.Py_NE, y) +pycompare(::Type{T}, x, ::typeof(< ), y) where {T} = _pycompare(T, x, C.Py_LT, y) +pycompare(::Type{T}, x, ::typeof(<=), y) where {T} = _pycompare(T, x, C.Py_LE, y) +pycompare(::Type{T}, x, ::typeof(> ), y) where {T} = _pycompare(T, x, C.Py_GT, y) +pycompare(::Type{T}, x, ::typeof(>=), y) where {T} = _pycompare(T, x, C.Py_GE, y) +pycompare(x, op, y) = pycompare(PyObject, x, op, y) +_pycompare(::Type{T}, x, op::Cint, y) where {T} = cpyop(T, (xo,yo)->C.PyObject_RichCompare(xo, yo, op), x, y) +_pycompare(::Type{Bool}, x, op::Cint, y) = checkm1(cpyop((xo,yo)->C.PyObject_RichCompareBool(xo, yo, op), x, y)) != 0 +export pycompare + +""" + pyeq([T=PyObject,] x, y) :: T + +Equivalent to `x == y` in Python. +""" +pyeq(::Type{T}, x, y) where {T} = pycompare(T, x, ==, y) +pyeq(x, y) = pycompare(x, ==, y) +export pyeq + +""" + pyne([T=PyObject,] x, y) :: T + +Equivalent to `x != y` in Python. +""" +pyne(::Type{T}, x, y) where {T} = pycompare(T, x, !=, y) +pyne(x, y) = pycompare(x, !=, y) +export pyne + +""" + pyge([T=PyObject,] x, y) :: T + +Equivalent to `x >= y` in Python. +""" +pyge(::Type{T}, x, y) where {T} = pycompare(T, x, >=, y) +pyge(x, y) = pycompare(x, >=, y) +export pyge + +""" + pygt([T=PyObject,] x, y) :: T + +Equivalent to `x > y` in Python. +""" +pygt(::Type{T}, x, y) where {T} = pycompare(T, x, >, y) +pygt(x, y) = pycompare(x, >, y) +export pygt + +""" + pyle([T=PyObject,] x, y) :: T + +Equivalent to `x <= y` in Python. +""" +pyle(::Type{T}, x, y) where {T} = pycompare(T, x, <=, y) +pyle(x, y) = pycompare(x, <=, y) +export pyle + +""" + pylt([T=PyObject,] x, y) :: T + +Equivalent to `x < y` in Python. +""" +pylt(::Type{T}, x, y) where {T} = pycompare(T, x, <, y) +pylt(x, y) = pycompare(x, <, y) +export pylt + +""" + pyadd([T=PyObject,] x, y) :: T + +Equivalent to `x + y` in Python. +""" +pyadd(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Add, x, y) +pyadd(x, y) = pyadd(PyObject, x, y) +export pyadd + +""" + pyiadd([T=typeof(x),] x, y) :: T + +`x = pyiadd(x, y)` is equivalent to `x += y` in Python. +""" +pyiadd(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceAdd, x, y) +pyiadd(x, y) = pyiadd(typeof(x), x, y) +export pyiadd + +""" + pysub([T=PyObject,] x, y) :: T + +Equivalent to `x - y` in Python. +""" +pysub(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Subtract, x, y) +pysub(x, y) = pysub(PyObject, x, y) +export pysub + +""" + pyisub([T=typeof(x),] x, y) :: T + +`x = pyisub(x, y)` is equivalent to `x -= y` in Python. +""" +pyisub(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceSubtract, x, y) +pyisub(x, y) = pyisub(typeof(x), x, y) +export pyisub + +""" + pymul([T=PyObject,] x, y) :: T + +Equivalent to `x * y` in Python. +""" +pymul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Multiply, x, y) +pymul(x, y) = pymul(PyObject, x, y) +export pymul -# exceptions and warnings -# NOTE: We import these directly from C, so that the error indicator is not inadvertantly -# reset when the lazy object is evaluated the first time (otherwise there can be problems -# using `pyerroccurred(t)`). -for n in [ - # exceptions - :BaseException, :Exception, :ArithmeticError, :AssertionError, :AttributeError, - :BlockingIOError, :BrokenPipeError, :BufferError, :ChildProcessError, - :ConnectionAbortedError, :ConnectionError, :ConnectionRefusedError, - :ConnectionResetError, :EOFError, :FileExistsError, :FileNotFoundError, - :FloatingPointError, :GeneratorExit, :ImportError, :IndentationError, :IndexError, - :InterruptedError, :IsADirectoryError, :KeyError, :KeyboardInterrupt, :LookupError, - :MemoryError, :ModuleNotFoundError, :NameError, :NotADirectoryError, - :NotImplementedError, :OSError, :OverflowError, :PermissionError, :ProcessLookupError, - :RecursionError, :ReferenceError, :RuntimeError, :StopAsyncIteration, :StopIteration, - :SyntaxError, :SystemError, :SystemExit, :TabError, :TimeoutError, :TypeError, - :UnboundLocalError, :UnicodeDecodeError, :UnicodeEncodeError, :UnicodeError, - :UnicodeTranslateError, :ValueError, :ZeroDivisionError, - # aliases - :EnvironmentError, :IOError, :WindowsError, - # warnings - :Warning, :BytesWarning, :DeprecationWarning, :FutureWarning, :ImportWarning, - :PendingDeprecationWarning, :ResourceWarning, :RuntimeWarning, :SyntaxWarning, - :UnicodeWarning, :UserWarning, +""" + pyimul([T=typeof(x),] x, y) :: T + +`x = pyimul(x, y)` is equivalent to `x *= y` in Python. +""" +pyimul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceMultiply, x, y) +pyimul(x, y) = pyimul(typeof(x), x, y) +export pyimul + +""" + pymatmul([T=PyObject,] x, y) :: T + +Equivalent to `x @ y` in Python. +""" +pymatmul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_MatrixMultiply, x, y) +pymatmul(x, y) = pymatmul(PyObject, x, y) +export pymatmul + +""" + pyimatmul([T=typeof(x),] x, y) :: T + +`x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. +""" +pyimatmul(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceMatrixMultiply, x, y) +pyimatmul(x, y) = pyimatmul(typeof(x), x, y) +export pyimatmul + +""" + pyfloordiv([T=PyObject,] x, y) :: T + +Equivalent to `x // y` in Python. +""" +pyfloordiv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_FloorDivide, x, y) +pyfloordiv(x, y) = pyfloordiv(PyObject, x, y) +export pyfloordiv + +""" + pyifloordiv([T=typeof(x),] x, y) :: T + +`x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. +""" +pyifloordiv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceFloorDivide, x, y) +pyifloordiv(x, y) = pyifloordiv(typeof(x), x, y) +export pyifloordiv + +""" + pytruediv([T=PyObject,] x, y) :: T + +Equivalent to `x / y` in Python. +""" +pytruediv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_TrueDivide, x, y) +pytruediv(x, y) = pytruediv(PyObject, x, y) +export pytruediv + +""" + pyitruediv([T=typeof(x),] x, y) :: T + +`x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. +""" +pyitruediv(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceTrueDivide, x, y) +pyitruediv(x, y) = pyitruediv(typeof(x), x, y) +export pyitruediv + +""" + pyrem([T=PyObject,] x, y) :: T + +Equivalent to `x % y` in Python. +""" +pyrem(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Remainder, x, y) +pyrem(x, y) = pyrem(PyObject, x, y) +export pyrem + +""" + pyirem([T=typeof(x),] x, y) :: T + +`x = pyirem(x, y)` is equivalent to `x %= y` in Python. +""" +pyirem(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceRemainder, x, y) +pyirem(x, y) = pyirem(typeof(x), x, y) +export pyirem + +""" + pydivmod([T=PyObject,] x, y) :: T + +Equivalent to `divmod(x, y)` in Python. +""" +pydivmod(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_DivMod, x, y) +pydivmod(x, y) = pydivmod(PyObject, x, y) +export pydivmod + +""" + pylshift([T=PyObject,] x, y) :: T + +Equivalent to `x << y` in Python. +""" +pylshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Lshift, x, y) +pylshift(x, y) = pylshift(PyObject, x, y) +export pylshift + +""" + pyilshift([T=typeof(x),] x, y) :: T + +`x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. +""" +pyilshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceLshift, x, y) +pyilshift(x, y) = pyilshift(typeof(x), x, y) +export pyilshift + +""" + pyrshift([T=PyObject,] x, y) :: T + +Equivalent to `x >> y` in Python. +""" +pyrshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Rshift, x, y) +pyrshift(x, y) = pyrshift(PyObject, x, y) +export pyrshift + +""" + pyirshift([T=typeof(x),] x, y) :: T + +`x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. +""" +pyirshift(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceRshift, x, y) +pyirshift(x, y) = pyirshift(typeof(x), x, y) +export pyirshift + +""" + pyand([T=PyObject,] x, y) :: T + +Equivalent to `x & y` in Python. +""" +pyand(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_And, x, y) +pyand(x, y) = pyand(PyObject, x, y) +export pyand + +""" + pyiand([T=typeof(x),] x, y) :: T + +`x = pyiand(x, y)` is equivalent to `x &= y` in Python. +""" +pyiand(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceAnd, x, y) +pyiand(x, y) = pyiand(typeof(x), x, y) +export pyiand + +""" + pyxor([T=PyObject,] x, y) :: T + +Equivalent to `x ^ y` in Python. +""" +pyxor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Xor, x, y) +pyxor(x, y) = pyxor(PyObject, x, y) +export pyxor + +""" + pyixor([T=typeof(x),] x, y) :: T + +`x = pyixor(x, y)` is equivalent to `x ^= y` in Python. +""" +pyixor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceXor, x, y) +pyixor(x, y) = pyixor(typeof(x), x, y) +export pyixor + +""" + pyor([T=PyObject,] x, y) :: T + +Equivalent to `x | y` in Python. +""" +pyor(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_Or, x, y) +pyor(x, y) = pyor(PyObject, x, y) +export pyor + +""" + pyior([T=typeof(x),] x, y) :: T + +`x = pyior(x, y)` is equivalent to `x |= y` in Python. +""" +pyior(::Type{T}, x, y) where {T} = cpyop(T, C.PyNumber_InPlaceOr, x, y) +pyior(x, y) = pyior(typeof(x), x, y) +export pyior + +""" + pypow([T=PyObject,] x, y, [z]) :: T + +Equivalent to `x**y` or `pow(x, y, z)` in Python. +""" +pypow(::Type{T}, x, y, z=C.PyObjectRef(C.Py_None())) where {T} = cpyop(T, C.PyNumber_Power, x, y, z) +pypow(x, y, z) = pypow(PyObject, x, y, z) +pypow(x, y) = pypow(PyObject, x, y) +export pypow + +""" + pyipow([T=typeof(x),] x, y, [z]) :: T + +`x = pyipow(x, y)` is equivalent to `x **= y` in Python. +""" +pyipow(::Type{T}, x, y, z=C.PyObjectRef(C.Py_None())) where {T} = cpyop(T, C.PyNumber_InPlacePower, x, y, z) +pyipow(x, y, z) = pyipow(typeof(x), x, y, z) +pyipow(x, y) = pyipow(typeof(x), x, y) +export pyipow + +""" + pyneg([T=typeof(x),] x) :: T + +Equivalent to `-x` in Python. +""" +pyneg(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Negative, x) +pyneg(x) = pyneg(typeof(x), x) +export pyneg + +""" + pypos([T=typeof(x),] x) :: T + +Equivalent to `+x` in Python. +""" +pypos(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Positive, x) +pypos(x) = pypos(typeof(x), x) +export pypos + +""" + pyabs([T=typeof(x),] x) :: T + +Equivalent to `abs(x)` in Python. +""" +pyabs(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Absolute, x) +pyabs(x) = pyabs(typeof(x), x) +export pyabs + +""" + pyinv([T=typeof(x),] x) :: T + +Equivalent to `-x` in Python. +""" +pyinv(::Type{T}, x) where {T} = cpyop(T, C.PyNumber_Invert, x) +pyinv(x) = pyinv(typeof(x), x) +export pyinv + +""" + pyiter([T=PyObject] x) :: T + +Equivalent to `iter(x)` in Python. +""" +pyiter(::Type{T}, x) where {T} = cpyop(T, C.PyObject_GetIter, x) +pyiter(x) = pyiter(PyObject, x) +export pyiter + +""" + pywith(f, o, d=nothing) + +Equivalent to `with o as x: f(x)` in Python, where `x` is a `PyObject`. + +On success, the value of `f(x)` is returned. +If an exception occurs but is suppressed then `d` is returned. +""" +function pywith(f, _o, d=nothing) + o = PyObject(_o) + t = pytype(o) + exit = t.__exit__ + value = t.__enter__(o) + exited = false + try + return f(value) + catch err + if err isa PyException + exited = true + if pytruth(exit(o, err.tref, err.vref, err.bref)) + rethrow() + else + return d + end + else + rethrow() + end + finally + exited || exit(o, pynone(), pynone(), pynone()) + end +end +export pywith + +""" + pytuple([T=PyObject,] [x]) :: T + +Create a Python `tuple` from the elements of `x`. +""" +pytuple(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyTuple_Type(), x) : C.PyTuple_FromIter(x)) +pytuple(::Type{T}) where {T} = checknullconvert(T, C.PyTuple_New(0)) +pytuple(x) = pytuple(PyObject, x) +pytuple() = pytuple(PyObject) +export pytuple + +""" + pylist([T=PyObject,] [x]) :: T + +Create a Python `list` from the elements of `x`. +""" +pylist(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyList_Type(), x) : C.PyList_FromIter(x)) +pylist(::Type{T}) where {T} = checknullconvert(T, C.PyList_New(0)) +pylist(x) = pylist(PyObject, x) +pylist() = pylist(PyObject) +export pylist + +""" + pycollist([T=PyObject,] x::AbstractArray) :: T + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of columns. +""" +pycollist(::Type{T}, x::AbstractArray) where {T} = ndims(x)==0 ? pyconvert(T, x[]) : pylist(T, pycollist(PyRef, y) for y in eachslice(x; dims=ndims(x))) +pycollist(x::AbstractArray) = pycollist(PyObject, x) +export pycollist + +""" + pyrowlist([T=PyObject,] x::AbstractArray) :: T + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of rows. +""" +pyrowlist(::Type{T}, x::AbstractArray) where {T} = ndims(x)==0 ? pyconvert(T, x[]) : pylist(T, pyrowlist(PyRef, y) for y in eachslice(x; dims=1)) +pyrowlist(x::AbstractArray) = pyrowlist(PyObject, x) +export pyrowlist + +""" + pyset([T=PyObject,] [x]) :: T + +Create a Python `set` from the elements of `x`. +""" +pyset(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PySet_Type(), x) : C.PySet_FromIter(x)) +pyset(::Type{T}) where {T} = checknullconvert(T, C.PySet_New(C_NULL)) +pyset(x) = pyset(PyObject, x) +pyset() = pyset(PyObject) +export pyset + +""" + pyfrozenset([T=PyObject,] [x]) :: T + +Create a Python `frozenset` from the elements of `x`. +""" +pyfrozenset(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyFrozenSet_Type(), x) : C.PyFrozenSet_FromIter(x)) +pyfrozenset(::Type{T}) where {T} = checknullconvert(T, C.PyFrozenSet_New(C_NULL)) +pyfrozenset(x) = pyfrozenset(PyObject, x) +pyfrozenset() = pyfrozenset(PyObject) +export pyfrozenset + +""" + pydict([T=PyObject,] [x]) :: T + +Create a Python `dict` from the key-value pairs in `x`. +""" +pydict(::Type{T}, x) where {T} = checknullconvert(T, ispyreftype(typeof(x)) ? C.PyObject_CallNice(C.PyDict_Type(), x) : C.PyDict_FromPairs(x)) +pydict(::Type{T}) where {T} = checknullconvert(T, C.PyDict_New()) +pydict(x) = pydict(PyObject, x) +pydict() = pydict(PyObject) +export pydict + +""" + pyslice([T=PyObject,] [start,] stop, [step]) :: T + +Equivalent to `slice(start, stop, step)` in Python (or `start:stop:step` while indexing). +""" +pyslice(::Type{T}, x) where {T} = cpyop(T, x->C.PySlice_New(C_NULL, x, C_NULL), x) +pyslice(::Type{T}, x, y) where {T} = cpyop(T, (x,y)->C.PySlice_New(x, y, C_NULL), x, y) +pyslice(::Type{T}, x, y, z) where {T} = cpyop(T, C.PySlice_New, x, y, z) +pyslice(x) = pyslice(PyObject, x) +pyslice(x, y) = pyslice(PyObject, x, y) +pyslice(x, y, z) = pyslice(PyObject, x, y, z) +export pyslice + +""" + pyellipsis([T=PyObject]) + +Equivalent to `Ellipsis` in Python (or `...` while indexing). +""" +pyellipsis(::Type{T}) where {T} = checknullconvert(T, C.PyEllipsis_New()) +pyellipsis() = pyellipsis(PyObject) +export pyellipsis + +### MULTIMEDIA DISPLAY + +const _py_mimes = [ + (MIME"text/html", "_repr_html_"), (MIME"text/markdown", "_repr_markdown_"), + (MIME"text/json", "_repr_json_"), (MIME"application/javascript", "_repr_javascript_"), + (MIME"application/pdf", "_repr_pdf_"), (MIME"image/jpeg", "_repr_jpeg_"), + (MIME"image/png", "_repr_png_"), (MIME"image/svg+xml", "_repr_svg_"), + (MIME"text/latex", "_repr_latex_") ] - j = Symbol(:py, lowercase(string(n))) - p = QuoteNode(Symbol(:PyExc_, n)) - @eval const $j = pylazyobject(() -> pyborrowedobject(unsafe_load(Ptr{CPyPtr}(C.pyglobal($p))))) - @eval export $j +const _py_mimetype = Union{map(first, _py_mimes)...} + +for (mime, method) in _py_mimes + T = istextmime(mime()) ? String : Vector{UInt8} + @eval begin + _py_mime_show(io::IO, mime::$mime, o) = begin + try + x = pycall(PyRef, pygetattr(PyRef, o, $method)) + pyis(x, pynone(PyRef)) || return write(io, pyconvert($T, x)) + catch + end + throw(MethodError(_py_mime_show, (io, mime, o))) + end + _py_mime_showable(::$mime, o) = begin + try + x = pycall(PyRef, pygetattr(PyRef, o, $method)) + if pyis(x, pynone(PyRef)) + false + else + pyconvert($T, x) + true + end + catch + false + end + end + end end diff --git a/src/bytearray.jl b/src/bytearray.jl deleted file mode 100644 index 705a6fa8..00000000 --- a/src/bytearray.jl +++ /dev/null @@ -1,8 +0,0 @@ -const pybytearraytype = pylazyobject(() -> pybuiltins.bytearray) -export pybytearraytype - -pyisbytearray(o::PyObject) = pytypecheck(o, pybytearraytype) -export pyisbytearray - -pybytearray(args...; opts...) = pybytearraytype(args...; opts...) -export pybytearray diff --git a/src/bytes.jl b/src/bytes.jl deleted file mode 100644 index 923005c5..00000000 --- a/src/bytes.jl +++ /dev/null @@ -1,16 +0,0 @@ -const pybytestype = pylazyobject(() -> pybuiltins.bytes) -export pybytestype - -pyisbytes(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_BYTES_SUBCLASS) -export pyisbytes - -pybytes(args...; opts...) = pybytestype(args...; opts...) -pybytes(x::Vector{UInt8}) = check(C.PyBytes_FromStringAndSize(pointer(x), length(x))) -export pybytes - -pybytes_asjuliastring(o::PyObject) = GC.@preserve o begin - buf = Ref{Ptr{Cchar}}() - len = Ref{C.Py_ssize_t}() - check(C.PyBytes_AsStringAndSize(o, buf, len)) - unsafe_string(buf[], len[]) -end diff --git a/src/collections.jl b/src/collections.jl deleted file mode 100644 index 375afb7b..00000000 --- a/src/collections.jl +++ /dev/null @@ -1,146 +0,0 @@ -for p in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, :Sized, :Callable, :Collection, :Sequence, :MutableSequence, :ByteString, :Set, :MutableSet, :Mapping, :MutableMapping, :MappingView, :ItemsView, :KeysView, :ValuesView, :Awaitable, :Coroutine, :AsyncIterable, :AsyncIterator, :AsyncGenerator] - j = Symbol(:py, lowercase(string(p)), :abc) - @eval const $j = pylazyobject(() -> pycollectionsabcmodule.$p) - @eval export $j -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T} - # check subclasses - if pyisinstance(o, pymappingabc) - r = pymapping_tryconvert(T, o) - r === PyConvertFail() || return r - elseif pyisinstance(o, pysequenceabc) - r = pysequence_tryconvert(T, o) - r === PyConvertFail() || return r - elseif pyisinstance(o, pysetabc) - r = pyabstractset_tryconvert(T, o) - r === PyConvertFail() || return r - end - # generic conversions - if (S = _typeintersect(T, PyIterable)) != Union{} - return S(o) - elseif (S = _typeintersect(T, Vector{PyObject})) != Union{} - r = S() - append!(r, PyIterable{PyObject}(o)) - elseif (S = _typeintersect(T, Vector)) != Union{} - r = S() - append!(r, PyIterable{eltype(r)}(o)) - elseif (S = _typeintersect(T, Set{PyObject})) != Union{} - r = S() - append!(r, PyIterable{PyObject}(o)) - elseif (S = _typeintersect(T, Set)) != Union{} - r = S() - append!(r, PyIterable{eltype(r)}(o)) - elseif (S = _typeintersect(T, Tuple)) != Union{} - pyiterable_tryconvert(S, o) - elseif (S = _typeintersect(T, CartesianIndex)) != Union{} - pyiterable_tryconvert(S, o) - elseif (S = _typeintersect(T, NamedTuple)) != Union{} - pyiterable_tryconvert(S, o) - else - tryconvert(T, PyIterable(o)) - end -end - -# TODO: make parts of this generated for type stability (i.e. the parts dealing with type logic) -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:Tuple} - # union? - if T isa Union - a = pyiterable_tryconvert(T.a, o) - b = pyiterable_tryconvert(T.b, o) - if typeof(a) == typeof(b) - return a - elseif a === PyConvertFail() - return b - elseif b === PyConvertFail() - return a - else - error("ambiguous conversion") - end - end - # flatten out any type vars - S = _type_flatten_tuple(T) :: DataType - # determine component types - if length(S.parameters) > 0 && Base.isvarargtype(S.parameters[end]) - nfixed = length(S.parameters) - 1 - vartype = S.parameters[end].body.parameters[1] - else - nfixed = length(S.parameters) - vartype = nothing - end - # convert components - xs = [] - for (i,xo) in enumerate(o) - if i ≤ nfixed - x = pytryconvert(S.parameters[i], xo) - x === PyConvertFail() ? (return x) : push!(xs, x) - elseif vartype === nothing - # too many values - return PyConvertFail() - else - x = pytryconvert(vartype, xo) - x === PyConvertFail() ? (return x) : push!(xs, x) - end - end - # check we got enough - length(xs) ≥ nfixed || return PyConvertFail() - # success! - tryconvert(T, Tuple(xs)) -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:CartesianIndex} - x = pyiterable_tryconvert(Tuple{Vararg{Int}}, o) - x === PyConvertFail() ? x : convert(T, CartesianIndex(x)) -end - -function pyiterable_tryconvert(::Type{CartesianIndex{N}}, o::PyObject) where {N} - x = pyiterable_tryconvert(NTuple{N,Int}, o) - x === PyConvertFail() ? x : CartesianIndex{N}(x) -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {T<:NamedTuple} - PyConvertFail() -end - -function pyiterable_tryconvert(::Type{T}, o::PyObject) where {names, T<:NamedTuple{names}} - x = pyiterable_tryconvert(NTuple{length(names), PyObject}, o) - x === PyConvertFail() ? x : convert(T, NamedTuple{names}(x)) -end - -function pyiterable_tryconvert(::Type{NamedTuple{names,Ts}}, o::PyObject) where {names, Ts<:Tuple} - x = pyiterable_tryconvert(Ts, o) - x === PyConvertFail() ? x : NamedTuple{names,Ts}(x) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, AbstractDict)) != Union{} - pymapping_tryconvert(S, o) - else - PyConvertFail() - end -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {T<:AbstractDict} - tryconvert(T, PyDict(o)) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {K, T<:AbstractDict{K}} - # often fails because e.g. convert(::Dict{String}, ::PyDict{String,Int}) fails - tryconvert(T, PyDict{K}(o)) -end - -function pymapping_tryconvert(::Type{T}, o::PyObject) where {K, V, T<:AbstractDict{K,V}} - tryconvert(T, PyDict{K,V}(o)) -end - -function pysequence_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, PyList)) != Union{} - S(o) - else - PyConvertFail() - end -end - -function pyabstractset_tryconvert(::Type{T}, o::PyObject) where {T} - PyConvertFail() -end diff --git a/src/complex.jl b/src/complex.jl deleted file mode 100644 index dbfcf5be..00000000 --- a/src/complex.jl +++ /dev/null @@ -1,24 +0,0 @@ -const pycomplextype = pylazyobject(() -> pybuiltins.complex) -export pycomplextype - -pycomplex(args...; opts...) = pycomplextype(args...; opts...) -pycomplex(x::Complex) = pycomplex(real(x), imag(x)) -export pycomplex - -pyiscomplex(o::PyObject) = pytypecheck(o, pycomplextype) -export pyiscomplex - -function pycomplex_tryconvert(::Type{T}, o::PyObject) where {T} - x = check(C.PyComplex_RealAsDouble(o), true) - y = check(C.PyComplex_ImagAsDouble(o), true) - z = Complex(x, y) - if (S = _typeintersect(T, Complex{Cdouble})) != Union{} - convert(S, z) - elseif (S = _typeintersect(T, Complex)) != Union{} - tryconvert(S, z) - elseif (S = _typeintersect(T, Real)) != Union{} - tryconvert(S, z) - else - tryconvert(T, z) - end -end diff --git a/src/convert.jl b/src/convert.jl deleted file mode 100644 index 57575635..00000000 --- a/src/convert.jl +++ /dev/null @@ -1,221 +0,0 @@ -function pyconvert(::Type{T}, o::PyObject) where {T} - r = pytryconvert(T, o) - r === PyConvertFail() ? error("cannot convert this Python `$(pytype(o).__name__)` to a Julia `$T`") : r -end -pyconvert(::Type{T}) where {T} = o -> pyconvert(T, o) -export pyconvert - -function pytryconvert(::Type{T}, o::PyObject) where {T} - # special cases - if T == PyObject - return PyObject(o) - elseif pyisjl(o) - return tryconvert(T, pyjlgetvalue(o)) - end - - # types - for t in pytype(o).__mro__ - n = "$(t.__module__).$(t.__name__)" - c = get(PYTRYCONVERT_TYPE_RULES, n, (T,o)->PyConvertFail()) - r = c(T, o) :: Union{T, PyConvertFail} - r === PyConvertFail() || return r - end - - # buffer/array interface - if (C.PyObject_CheckBuffer(o) != 0) || (pyhasattr(o, "__array__") && !pyisnone(o.__array__)) || (pyhasattr(o, "__array_interface__") && !pyisnone(o.__array_interface__)) - r = tryconvert(T, PyArray(o)) - r === PyConvertFail() || return r - end - # iterable interface (includes sequences, mappings and sets) - if pyisinstance(o, pyiterableabc) - r = pyiterable_tryconvert(T, o) :: Union{T, PyConvertFail} - r === PyConvertFail() || return r - end - # TODO: IO interface - # TODO: number interface - - # so that T=Any always succeeds - if o isa T - return o - end - - # failure - return PyConvertFail() -end -pytryconvert(::Type{T}) where {T} = o -> pytryconvert(T, o) -export pytryconvert - -const PYTRYCONVERT_TYPE_RULES = Dict{String,Function}( - "builtins.NoneType" => pynone_tryconvert, - "builtins.bool" => pybool_tryconvert, - "builtins.str" => pystr_tryconvert, - "builtins.int" => pyint_tryconvert, - "builtins.float" => pyfloat_tryconvert, - "builtins.complex" => pycomplex_tryconvert, - "builtins.Fraction" => pyfraction_tryconvert, - "builtins.range" => pyrange_tryconvert, - "builtins.tuple" => pytuple_tryconvert, - "pandas.core.frame.DataFrame" => pypandasdataframe_tryconvert, - # TODO: datetime, date, time - # NOTE: we don't need to include standard containers here because we can access them via standard interfaces (Sequence, Mapping, Buffer, etc.) -) - -Base.convert(::Type{T}, o::PyObject) where {T} = pyconvert(T, o) -Base.convert(::Type{Any}, o::PyObject) = o - -### SPECIAL CONVERSIONS - -@generated _eltype(o) = try eltype(o); catch; missing; end -_eltype(o::Type) = missing - -@generated _keytype(o) = try keytype(o); catch; missing; end -_keytype(o::Base.RefValue) = Tuple{} -_keytype(o::NamedTuple) = Union{Symbol,Int} -_keytype(o::Tuple) = Int -_keytype(o::Type) = missing - -@generated _valtype(o, k...) = try valtype(o); catch; missing; end -_valtype(o::NamedTuple, k::Int) = fieldtype(typeof(o), k) -_valtype(o::NamedTuple, k::Symbol) = fieldtype(typeof(o), k) -_valtype(o::Tuple, k::Int) = fieldtype(typeof(o), k) -_valtype(o::Type) = missing - -hasmultiindex(o) = true -hasmultiindex(o::Type) = true -hasmultiindex(o::AbstractDict) = false -hasmultiindex(o::NamedTuple) = false -hasmultiindex(o::Tuple) = false - -""" - pyconvert_element(o, v::PyObject) - -Convert `v` to be of the right type to be an element of `o`. -""" -pytryconvert_element(o, v) = pytryconvert(_eltype(o)===missing ? Any : _eltype(o), v) -pyconvert_element(args...) = - let r = pytryconvert_element(args...) - r === PyConvertFail() ? error("cannot convert this to an element") : r - end - -""" - pytryconvert_indices(o, k::PyObject) - -Convert `k` to a tuple of indices for `o`. -""" -function pytryconvert_indices(o, k) - if _keytype(o) !== missing - i = pytryconvert(_keytype(o), k) - i === PyConvertFail() ? PyConvertFail() : (i,) - elseif hasmultiindex(o) && pyistuple(k) - Tuple(pyconvert(Any, x) for x in k) - else - (pyconvert(Any, k),) - end -end -pyconvert_indices(args...) = - let r = pytryconvert_indices(args...) - r === PyConvertFail() ? error("cannot convert this to indices") : r - end - -""" - pyconvert_value(o, v::PyObject, k...) - -Convert `v` to be of the right type to be a value of `o` at indices `k`. -""" -pytryconvert_value(o, v, k...) = pytryconvert(_valtype(o)===missing ? Any : _valtype(o), v) -pyconvert_value(args...) = - let r = pytryconvert_value(args...) - r === PyConvertFail() ? error("cannot convert this to a value") : r - end - -""" - pyconvert_args(NamedTuple{argnames, argtypes}, args, kwargs=nothing; defaults=nothing, numpos=len(names), numposonly=0) - -Parse the Python tuple `args` and optionally Python dict `kwargs` as function arguments, returning a `NamedTuple{names,types}`. - -- `defaults` is a named tuple of default parameter values for optional arguments. -- `numpos` is the number of possibly-positional arguments. -- `numposonly` is the number of positional-only arguments. -""" -@generated function pyconvert_args( - ::Type{NamedTuple{argnames,argtypes}}, - args::PyObject, - kwargs::Union{PyObject,Nothing}=nothing; - defaults::Union{NamedTuple,Nothing}=nothing, - numposonly::Integer=0, - numpos::Integer=length(argnames), -) where {argnames, argtypes} - code = [] - argvars = [] - push!(code, quote - 0 ≤ numposonly ≤ numpos ≤ length(argnames) || error("require 0 ≤ numposonly ≤ numpos ≤ numargs") - nargs = pylen(args) - nargs ≤ numpos || pythrow(pytypeerror("at most $numpos positional arguments expected, got $nargs")) - end) - for (i,(argname,argtype)) in enumerate(zip(argnames,argtypes.parameters)) - argvar = gensym() - push!(argvars, argvar) - push!(code, quote - $argvar = - if $i ≤ numpos && $i ≤ nargs - kwargs !== nothing && $(string(argname)) in kwargs && pythrow(pytypeerror($("argument '$argname' got multiple values"))) - let r = pytryconvert($argtype, args[$(i-1)]) - r===PyConvertFail() ? pythrow(pytypeerror($("argument '$argname' has wrong type"))) : r - end - elseif kwargs !== nothing && $i > numposonly && $(string(argname)) in kwargs - let r = pytryconvert($argtype, kwargs.pop($(string(argname)))) - r===PyConvertFail() ? pythrow(pytypeerror($("argument '$argname' has wrong type"))) : r - end - elseif defaults !== nothing && haskey(defaults, $(QuoteNode(argname))) - convert($argtype, defaults[$(QuoteNode(argname))]) - else - pythrow(pytypeerror($("argument '$argname' not given"))) - end - end) - end - push!(code, quote - kwargs !== nothing && pylen(kwargs) > 0 && pythrow(pytypeerror("invalid keyword arguments: $(join(kwargs, ", "))")) - NamedTuple{$argnames, $argtypes}(($(argvars...),)) - end) - ex = Expr(:block, code...) -end - -macro pyconvert_args(spec, args, kwargs=nothing) - spec isa Expr && spec.head == :tuple || @goto error - argnames = [] - argtypes = [] - defaults = [] - numposonly = 0 - numpos = -1 - for argspec in spec.args - if argspec === :* - numpos = length(argnames) - continue - elseif argspec == :/ - numposonly = length(argnames) - continue - end - if argspec isa Expr && argspec.head == :(=) - argspec, dflt = argspec.args - else - dflt = nothing - end - if argspec isa Expr && argspec.head == :(::) - argspec, tp = argspec.args - else - tp = PyObject - end - if argspec isa Symbol - nm = argspec - else - @goto error - end - push!(argnames, nm) - push!(argtypes, tp) - dflt === nothing || push!(defaults, nm => dflt) - end - numpos < 0 && (numpos = length(argnames)) - return :(pyconvert_args(NamedTuple{($(map(QuoteNode, argnames)...),),Tuple{$(map(esc, argtypes)...)}}, $(esc(args)), $(esc(kwargs)), defaults=NamedTuple{($([QuoteNode(n) for (n,d) in defaults]...),)}(($([esc(d) for (n,d) in defaults]...),)), numpos=$numpos, numposonly=$numposonly)) - @label error - error("invalid argument specification") -end diff --git a/src/cpython.jl b/src/cpython.jl deleted file mode 100644 index c5f0582a..00000000 --- a/src/cpython.jl +++ /dev/null @@ -1,511 +0,0 @@ -module CPython - - using Libdl - using ..Python: CONFIG - using Base: @kwdef - using UnsafePointers: UnsafePtr - - @enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 - - const Py_LT = Cint(0) - const Py_LE = Cint(1) - const Py_EQ = Cint(2) - const Py_NE = Cint(3) - const Py_GT = Cint(4) - const Py_GE = Cint(5) - - const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments - const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs - const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) - const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) - const Py_METH_CLASS = 0x0010 # for class methods - const Py_METH_STATIC = 0x0020 # for static methods - - const Py_T_SHORT =0 - const Py_T_INT =1 - const Py_T_LONG =2 - const Py_T_FLOAT =3 - const Py_T_DOUBLE =4 - const Py_T_STRING =5 - const Py_T_OBJECT =6 - const Py_T_CHAR =7 - const Py_T_BYTE =8 - const Py_T_UBYTE =9 - const Py_T_USHORT =10 - const Py_T_UINT =11 - const Py_T_ULONG =12 - const Py_T_STRING_INPLACE =13 - const Py_T_BOOL =14 - const Py_T_OBJECT_EX =16 - const Py_T_LONGLONG =17 # added in Python 2.5 - const Py_T_ULONGLONG =18 # added in Python 2.5 - const Py_T_PYSSIZET =19 # added in Python 2.6 - const Py_T_NONE =20 # added in Python 3.0 - - const Py_READONLY = 1 - const Py_READ_RESTRICTED = 2 - const Py_WRITE_RESTRICTED = 4 - const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) - - const PyBUF_MAX_NDIM = 64 - - # Flags for getting buffers - const PyBUF_SIMPLE = 0x0 - const PyBUF_WRITABLE = 0x0001 - const PyBUF_WRITEABLE = PyBUF_WRITABLE - const PyBUF_FORMAT = 0x0004 - const PyBUF_ND = 0x0008 - const PyBUF_STRIDES = (0x0010 | PyBUF_ND) - const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) - const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) - const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) - const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) - - const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) - const PyBUF_CONTIG_RO = (PyBUF_ND) - - const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) - const PyBUF_STRIDED_RO = (PyBUF_STRIDES) - - const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) - const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) - - const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) - const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) - - const PyBUF_READ = 0x100 - const PyBUF_WRITE = 0x200 - - # Python 2.7 - const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) - const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) - const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 - const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) - const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) - const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) - const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) - const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) - const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) - const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) - const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) - const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) - - # Python 3.0+ has only these: - const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) - const Py_TPFLAGS_BASETYPE = (0x00000001<<10) - const Py_TPFLAGS_READY = (0x00000001<<12) - const Py_TPFLAGS_READYING = (0x00000001<<13) - const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) - const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) - const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) - const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) - const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) - const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) - const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) - const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) - const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) - const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) - const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) - const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) - const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) - - # only use this if we have the stackless extension - const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) - - const Py_hash_t = Cssize_t - const Py_ssize_t = Cssize_t - - @kwdef struct Py_complex - real :: Cdouble = 0.0 - imag :: Cdouble = 0.0 - end - - @kwdef struct PyObject - # assumes _PyObject_HEAD_EXTRA is empty - refcnt :: Py_ssize_t = 0 - type :: Ptr{PyObject} = C_NULL - end - - const PyPtr = Ptr{PyObject} - - @kwdef struct PyVarObject - ob_base :: PyObject = PyObject() - size :: Py_ssize_t = 0 - end - - @kwdef struct PyMethodDef - name :: Cstring = C_NULL - meth :: Ptr{Cvoid} = C_NULL - flags :: Cint = 0 - doc :: Cstring = C_NULL - end - - @kwdef struct PyGetSetDef - name :: Cstring = C_NULL - get :: Ptr{Cvoid} = C_NULL - set :: Ptr{Cvoid} = C_NULL - doc :: Cstring = C_NULL - closure :: Ptr{Cvoid} = C_NULL - end - - @kwdef struct PyMemberDef - name :: Cstring = C_NULL - typ :: Cint = 0 - offset :: Py_ssize_t = 0 - flags :: Cint = 0 - doc :: Cstring = C_NULL - end - - @kwdef struct PyNumberMethods - add :: Ptr{Cvoid} = C_NULL # (o,o)->o - subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o - power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - negative :: Ptr{Cvoid} = C_NULL # (o)->o - positive :: Ptr{Cvoid} = C_NULL # (o)->o - absolute :: Ptr{Cvoid} = C_NULL # (o)->o - bool :: Ptr{Cvoid} = C_NULL # (o)->Cint - invert :: Ptr{Cvoid} = C_NULL # (o)->o - lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - and :: Ptr{Cvoid} = C_NULL # (o,o)->o - xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - or :: Ptr{Cvoid} = C_NULL # (o,o)->o - int :: Ptr{Cvoid} = C_NULL # (o)->o - _reserved :: Ptr{Cvoid} = C_NULL - float :: Ptr{Cvoid} = C_NULL # (o)->o - inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o - inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o - floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o - index :: Ptr{Cvoid} = C_NULL # (o)->o - matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o - end - - @kwdef struct PySequenceMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - _was_item :: Ptr{Cvoid} = C_NULL - ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint - _was_ass_slice :: Ptr{Cvoid} = C_NULL - contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint - inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o - inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o - end - - @kwdef struct PyMappingMethods - length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t - subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o - ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint - end - - @kwdef struct PyBufferProcs - get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint - release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid - end - - @kwdef struct Py_buffer - buf :: Ptr{Cvoid} = C_NULL - obj :: Ptr{Cvoid} = C_NULL - len :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - readonly :: Cint = 0 - ndim :: Cint = 0 - format :: Cstring = C_NULL - shape :: Ptr{Py_ssize_t} = C_NULL - strides :: Ptr{Py_ssize_t} = C_NULL - suboffsets :: Ptr{Py_ssize_t} = C_NULL - internal :: Ptr{Cvoid} = C_NULL - end - - @kwdef struct PyTypeObject - ob_base :: PyVarObject = PyVarObject() - name :: Cstring = C_NULL - - basicsize :: Py_ssize_t = 0 - itemsize :: Py_ssize_t = 0 - - dealloc :: Ptr{Cvoid} = C_NULL - vectorcall_offset :: Py_ssize_t = 0 - getattr :: Ptr{Cvoid} = C_NULL - setattr :: Ptr{Cvoid} = C_NULL - as_async :: Ptr{Cvoid} = C_NULL - repr :: Ptr{Cvoid} = C_NULL - - as_number :: Ptr{PyNumberMethods} = C_NULL - as_sequence :: Ptr{PySequenceMethods} = C_NULL - as_mapping :: Ptr{PyMappingMethods} = C_NULL - - hash :: Ptr{Cvoid} = C_NULL - call :: Ptr{Cvoid} = C_NULL - str :: Ptr{Cvoid} = C_NULL - getattro :: Ptr{Cvoid} = C_NULL - setattro :: Ptr{Cvoid} = C_NULL - - as_buffer :: Ptr{PyBufferProcs} = C_NULL - - flags :: Culong = 0 - - doc :: Cstring = C_NULL - - traverse :: Ptr{Cvoid} = C_NULL - - clear :: Ptr{Cvoid} = C_NULL - - richcompare :: Ptr{Cvoid} = C_NULL - - weaklistoffset :: Py_ssize_t = 0 - - iter :: Ptr{Cvoid} = C_NULL - iternext :: Ptr{Cvoid} = C_NULL - - methods :: Ptr{PyMethodDef} = C_NULL - members :: Ptr{PyMemberDef} = C_NULL - getset :: Ptr{PyGetSetDef} = C_NULL - base :: PyPtr = C_NULL - dict :: PyPtr = C_NULL - descr_get :: Ptr{Cvoid} = C_NULL - descr_set :: Ptr{Cvoid} = C_NULL - dictoffset :: Py_ssize_t = 0 - init :: Ptr{Cvoid} = C_NULL - alloc :: Ptr{Cvoid} = C_NULL - new :: Ptr{Cvoid} = C_NULL - free :: Ptr{Cvoid} = C_NULL - is_gc :: Ptr{Cvoid} = C_NULL - bases :: PyPtr = C_NULL - mro :: PyPtr = C_NULL - cache :: PyPtr = C_NULL - subclasses :: PyPtr = C_NULL - weaklist :: PyPtr = C_NULL - del :: Ptr{Cvoid} = C_NULL - - version_tag :: Cuint = 0 - - finalize :: Ptr{Cvoid} = C_NULL - vectorcall :: Ptr{Cvoid} = C_NULL - - end - - const PyTypePtr = Ptr{PyTypeObject} - - pyglobal(name) = dlsym(CONFIG.libptr, name) - - macro cdef(name, rettype, argtypes) - name isa QuoteNode && name.value isa Symbol || error("name must be a symbol, got $name") - jname = esc(name.value) - refname = esc(Symbol(name.value, :_funcptr)) - name = esc(name) - rettype = esc(rettype) - argtypes isa Expr && argtypes.head==:tuple || error("argtypes must be a tuple, got $argtypes") - nargs = length(argtypes.args) - argtypes = esc(argtypes) - args = [gensym() for i in 1:nargs] - quote - const $refname = Ref(C_NULL) - function $jname($(args...)) - ptr = $refname[] - if ptr == C_NULL - ptr = $refname[] = pyglobal($name) - end - ccall(ptr, $rettype, $argtypes, $(args...)) - end - end - end - - @cdef :Py_Initialize Cvoid () - @cdef :Py_InitializeEx Cvoid (Cint,) - @cdef :Py_Finalize Cvoid () - @cdef :Py_FinalizeEx Cint () - @cdef :Py_AtExit Cint (Ptr{Cvoid},) - @cdef :Py_IsInitialized Cint () - - @cdef :Py_SetPythonHome Cvoid (Cwstring,) - @cdef :Py_SetProgramName Cvoid (Cwstring,) - @cdef :Py_GetVersion Cstring () - - @cdef :Py_IncRef Cvoid (PyPtr,) - @cdef :Py_DecRef Cvoid (PyPtr,) - - @cdef :PyEval_SaveThread Ptr{Cvoid} () - @cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) - - @cdef :PyGILState_Ensure PyGILState_STATE () - @cdef :PyGILState_Release Cvoid (PyGILState_STATE,) - - @cdef :PyImport_ImportModule PyPtr (Cstring,) - @cdef :PyImport_Import PyPtr (PyPtr,) - - @cdef :PyErr_Occurred PyPtr () - @cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) - @cdef :PyErr_Clear Cvoid () - @cdef :PyErr_SetNone Cvoid (PyPtr,) - @cdef :PyErr_SetString Cvoid (PyPtr, Cstring) - @cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) - @cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) - @cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) - @cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) - - @cdef :_PyObject_New PyPtr (PyPtr,) - @cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) - @cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) - @cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) - @cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) - @cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) - @cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) - @cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) - @cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) - @cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) - @cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) - @cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) - @cdef :PyObject_Repr PyPtr (PyPtr,) - @cdef :PyObject_ASCII PyPtr (PyPtr,) - @cdef :PyObject_Str PyPtr (PyPtr,) - @cdef :PyObject_Bytes PyPtr (PyPtr,) - @cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) - @cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) - @cdef :PyObject_Hash Py_hash_t (PyPtr,) - @cdef :PyObject_IsTrue Cint (PyPtr,) - @cdef :PyObject_Length Py_ssize_t (PyPtr,) - @cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) - @cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_DelItem Cint (PyPtr, PyPtr) - @cdef :PyObject_Dir PyPtr (PyPtr,) - @cdef :PyObject_GetIter PyPtr (PyPtr,) - @cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) - - @cdef :PySequence_Contains Cint (PyPtr, PyPtr) - - @cdef :PyInstanceMethod_New PyPtr (PyPtr,) - - @cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) - @cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) - - @cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) - @cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) - - @cdef :PyTuple_New PyPtr (Py_ssize_t,) - @cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) - - @cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) - @cdef :PyType_Ready Cint (PyPtr,) - - @cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyNumber_Negative PyPtr (PyPtr,) - @cdef :PyNumber_Positive PyPtr (PyPtr,) - @cdef :PyNumber_Absolute PyPtr (PyPtr,) - @cdef :PyNumber_Invert PyPtr (PyPtr,) - @cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_And PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) - @cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) - @cdef :PyNumber_Long PyPtr (PyPtr,) - @cdef :PyNumber_Float PyPtr (PyPtr,) - @cdef :PyNumber_Index PyPtr (PyPtr,) - - @cdef :PyIter_Next PyPtr (PyPtr,) - - @cdef :PyLong_FromLongLong PyPtr (Clonglong,) - @cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) - @cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) - @cdef :PyLong_AsLongLong Clonglong (PyPtr,) - @cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) - - @cdef :PyFloat_FromDouble PyPtr (Cdouble,) - @cdef :PyFloat_AsDouble Cdouble (PyPtr,) - - @cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) - @cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) - - @cdef :PyList_New PyPtr (Py_ssize_t,) - @cdef :PyList_Append Cint (PyPtr, PyPtr) - @cdef :PyList_AsTuple PyPtr (PyPtr,) - - @cdef :PyDict_New PyPtr () - @cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) - @cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) - @cdef :PyDict_DelItemString Cint (PyPtr, Cstring) - - Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] - Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] - Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) - Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) - Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) - - PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] - PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) - PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) - - function PyObject_CheckBuffer(o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - p != C_NULL && p.get[] != C_NULL - end - - function PyObject_GetBuffer(o, b, flags) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if p == C_NULL || p.get[] == C_NULL - PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") - return Cint(-1) - end - ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) - end - - function PyBuffer_Release(_b) - b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) - o = b.obj[] - o == C_NULL && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (p != C_NULL && p.release[] != C_NULL) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) - end - b.obj[] = C_NULL - Py_DecRef(o) - return - end - - function PyOS_RunInputHook() - hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) - hook == C_NULL || ccall(hook, Cint, ()) - nothing - end - -end diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl new file mode 100644 index 00000000..06aa3db0 --- /dev/null +++ b/src/cpython/CPython.jl @@ -0,0 +1,218 @@ +module CPython + +using Libdl +import ..Python: CONFIG, isnull, ism1, PYERR, NOTIMPLEMENTED, _typeintersect, tryconvert, ispyreftype, pyptr, putresult, takeresult, CACHE, Python +using Base: @kwdef +using UnsafePointers: UnsafePtr + +pyglobal(name) = dlsym(CONFIG.libptr, name) +pyglobal(r::Ref{Ptr{T}}, name) where {T} = (p=r[]; if isnull(p); p=r[]=Ptr{T}(pyglobal(name)); end; p) +pyloadglobal(r::Ref{Ptr{T}}, name) where {T} = (p=r[]; if isnull(p); p=r[]=unsafe_load(Ptr{Ptr{T}}(pyglobal(name))) end; p) + +macro cdef(name, rettype, argtypes) + name isa QuoteNode && name.value isa Symbol || error("name must be a symbol, got $name") + jname = esc(name.value) + refname = esc(Symbol(name.value, :__ref)) + name = esc(name) + rettype = esc(rettype) + argtypes isa Expr && argtypes.head==:tuple || error("argtypes must be a tuple, got $argtypes") + nargs = length(argtypes.args) + argtypes = esc(argtypes) + args = [gensym() for i in 1:nargs] + quote + const $refname = Ref(C_NULL) + $jname($(args...)) = ccall(pyglobal($refname, $name), $rettype, $argtypes, $(args...)) + end +end + +include("consts.jl") +include("fundamentals.jl") +include("none.jl") +include("object.jl") +include("sequence.jl") +include("mapping.jl") +include("method.jl") +include("str.jl") +include("bytes.jl") +include("tuple.jl") +include("type.jl") +include("number.jl") +include("iter.jl") +include("bool.jl") +include("int.jl") +include("float.jl") +include("complex.jl") +include("list.jl") +include("dict.jl") +include("set.jl") +include("buffer.jl") +include("collections.jl") +include("range.jl") +include("ctypes.jl") +include("numpy.jl") +include("slice.jl") +include("fraction.jl") +include("newtype.jl") +include("juliaerror.jl") +include("juliabase.jl") +include("juliaraw.jl") +include("juliaany.jl") +include("juliaiterator.jl") +include("juliatype.jl") +include("juliadict.jl") +include("juliaarray.jl") +include("juliavector.jl") +include("juliamodule.jl") +include("arg.jl") + +__init__() = begin + PyObject_TryConvert_AddRules("builtins.NoneType", [ + (Nothing, PyNone_TryConvertRule_nothing, 100), + (Missing, PyNone_TryConvertRule_missing), + ]) + PyObject_TryConvert_AddRules("builtins.bool", [ + (Bool, PyBool_TryConvertRule_bool, 100), + ]) + PyObject_TryConvert_AddRules("numbers.Integral", [ + (Integer, PyLongable_TryConvertRule_integer, 100), + (Rational, PyLongable_TryConvertRule_tryconvert), + (Real, PyLongable_TryConvertRule_tryconvert), + (Number, PyLongable_TryConvertRule_tryconvert), + (Any, PyLongable_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.float", [ + (Float64, PyFloatable_TryConvertRule_convert, 100) + ]) + PyObject_TryConvert_AddRules("numbers.Real", [ + (Float64, PyFloatable_TryConvertRule_convert), + (BigFloat, PyFloatable_TryConvertRule_convert), + (Float32, PyFloatable_TryConvertRule_convert), + (Float16, PyFloatable_TryConvertRule_convert), + (AbstractFloat, PyFloatable_TryConvertRule_tryconvert), + (Real, PyFloatable_TryConvertRule_tryconvert), + (Number, PyFloatable_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.complex", [ + (Complex{Float64}, PyComplexable_TryConvertRule_convert, 100), + ]) + PyObject_TryConvert_AddRules("numbers.Complex", [ + (Complex{Float64}, PyComplexable_TryConvertRule_convert), + (Complex{BigFloat}, PyComplexable_TryConvertRule_convert), + (Complex{Float32}, PyComplexable_TryConvertRule_convert), + (Complex{Float16}, PyComplexable_TryConvertRule_convert), + (Complex{T} where {T<:AbstractFloat}, PyComplexable_TryConvertRule_tryconvert), + (Complex{T} where {T<:Real}, PyComplexable_TryConvertRule_tryconvert), + (Number, PyComplexable_TryConvertRule_tryconvert), + ]) + PyObject_TryConvert_AddRules("builtins.bytes", [ + (Vector{UInt8}, PyBytes_TryConvertRule_vector), + (Vector{Int8}, PyBytes_TryConvertRule_vector), + (String, PyBytes_TryConvertRule_string), + ]) + PyObject_TryConvert_AddRules("builtins.str", [ + (String, PyUnicode_TryConvertRule_string, 100), + (Symbol, PyUnicode_TryConvertRule_symbol), + (Char, PyUnicode_TryConvertRule_char), + (Vector{UInt8}, PyUnicode_TryConvertRule_vector), + (Vector{Int8}, PyUnicode_TryConvertRule_vector), + ]) + PyObject_TryConvert_AddRules("builtins.tuple", [ + (Tuple, PyIterable_ConvertRule_tuple, 100), + ]) + PyObject_TryConvert_AddRules("builtins.range", [ + (StepRange{T,S} where {T<:Integer, S<:Integer}, PyRange_TryConvertRule_steprange, 100), + (UnitRange{T} where {T<:Integer}, PyRange_TryConvertRule_unitrange), + ]) + PyObject_TryConvert_AddRules("collections.abc.Iterable", [ + (Vector, PyIterable_ConvertRule_vector), + (Set, PyIterable_ConvertRule_set), + (Tuple, PyIterable_ConvertRule_tuple), + (Pair, PyIterable_ConvertRule_pair), + ]) + PyObject_TryConvert_AddRules("collections.abc.Sequence", [ + (Vector, PyIterable_ConvertRule_vector), + ]) + PyObject_TryConvert_AddRules("collections.abc.Set", [ + (Set, PyIterable_ConvertRule_set), + ]) + PyObject_TryConvert_AddRules("collections.abc.Mapping", [ + (Dict, PyMapping_ConvertRule_dict), + ]) + PyObject_TryConvert_AddRules("julia.ValueBase", [ + (Any, PyJuliaValue_TryConvert_any, 200), + ]) + PyObject_TryConvert_AddExtraTypes([ + PyIterableABC_Type, + PyCallableABC_Type, + PySequenceABC_Type, + PyMappingABC_Type, + PySetABC_Type, + PyNumberABC_Type, + PyComplexABC_Type, + PyRealABC_Type, + PyRationalABC_Type, + PyIntegralABC_Type, + ]) + + ### ctypes + for (p,T) in [("char", Cchar), ("wchar", Cwchar_t), ("byte", Cchar), ("ubyte", Cuchar), + ("short", Cshort), ("ushort", Cushort), ("int", Cint), ("uint", Cuint), + ("long", Clong), ("ulong", Culong), ("longlong", Culonglong), ("size_t", Csize_t), + ("ssize_t", Cssize_t), ("float", Cfloat), ("double", Cdouble), #=("longdouble", ???),=# + ("char_p", Ptr{Cchar}), ("wchar_p", Ptr{Cwchar_t}), ("void_p", Ptr{Cvoid})] + isptr = occursin("_p", p) + isfloat = occursin("float", p) || occursin("double", p) + isint = !(isfloat || isptr) + isreal = isint || isfloat + PyObject_TryConvert_AddRules("ctypes.c_$p", [ + (p=="char_p" ? Cstring : p=="wchar_p" ? Cwstring : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (T, PySimpleCData_TryConvert_value{T,false}()), + (isint ? Integer : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isint ? Rational : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Float64 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? BigFloat : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? Float32 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? Float16 : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (isreal ? AbstractFloat : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Real : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isreal ? Number : Union{}, PySimpleCData_TryConvert_value{T,true}()), + (isptr ? Ptr : Union{}, PySimpleCData_TryConvert_value{T,false}()), + (Any, PySimpleCData_TryConvert_value{T,true}()), + ]) + end + + ### numpy + # TODO: Compound types + # TODO: datetime64, timedelta64 + for (p,T) in [("int8", Int8), ("int16", Int16), ("int32", Int32), ("int64", Int64), + ("int128", Int128), ("uint8", UInt8), ("uint16", UInt16), ("uint32", UInt32), + ("uint64", UInt64), ("uint128", UInt128), ("float16", Float16), ("float32", Float32), + ("float64", Float64), ("complex32", Complex{Float16}), + ("complex64", Complex{Float32}), ("complex128", Complex{Float64})] + isint = occursin("int", p) + isfloat = occursin("float", p) + iscomplex = occursin("complex", p) + isreal = isint || isfloat + PyObject_TryConvert_AddRules("numpy.$p", [ + (T, PyNumpySimpleData_TryConvert_value{T,false}(), 100), + (isint ? Integer : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isint ? Rational : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isreal ? Float64 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? BigFloat : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? Float32 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? Float16 : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (isreal ? AbstractFloat : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (isreal ? Real : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (iscomplex ? Complex{Float64} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{BigFloat} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{Float32} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? Complex{Float16} : Union{}, PyNumpySimpleData_TryConvert_value{T,false}()), + (iscomplex ? (Complex{T} where {T<:AbstractFloat}) : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (iscomplex ? (Complex{T} where {T<:Real}) : Union{}, PyNumpySimpleData_TryConvert_value{T,true}()), + (Number, PyNumpySimpleData_TryConvert_value{T,true}()), + (Any, PyNumpySimpleData_TryConvert_value{T,true}()), + ]) + end +end + +end diff --git a/src/cpython/arg.jl b/src/cpython/arg.jl new file mode 100644 index 00000000..9401698e --- /dev/null +++ b/src/cpython/arg.jl @@ -0,0 +1,118 @@ +PyArg_CheckNumArgsEq(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m == n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsGe(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m ≥ n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes at least $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsLe(name::String, args::PyPtr, n::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if m ≤ n + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes at most $n arguments ($m given)") + return -1 + end +end + +PyArg_CheckNumArgsBetween(name::String, args::PyPtr, n0::Integer, n1::Integer) = begin + m = isnull(args) ? 0 : PyTuple_Size(args) + if n0 ≤ m ≤ n1 + return 0 + else + PyErr_SetString(PyExc_TypeError(), "$name() takes $n0 to $n1 arguments ($m given)") + return -1 + end +end + +PyArg_CheckNoKwargs(name::String, kwargs::PyPtr) = begin + if isnull(kwargs) || PyObject_Length(kwargs) == 0 + return 0 + else + it = PyObject_GetIter(kwargs) + isnull(it) && return -1 + argnames = String[] + while true + argnameo = PyIter_Next(it) + if !isnull(argnameo) + argname = PyUnicode_AsString(argnameo) + Py_DecRef(argnameo) + isempty(argname) && PyErr_IsSet() && (Py_DecRef(it); return -1) + push!(argnames, argname) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + break + end + end + PyErr_SetString(PyExc_TypeError(), "$name() got unexpected keyword arguments: $(join(["'$n'" for n in argnames], ", "))") + return -1 + end +end + +struct NODEFAULT end + +PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}) = + if i !== nothing && !isnull(args) && 0 ≤ i < PyTuple_Size(args) + return PyTuple_GetItem(args, i) + elseif k !== nothing && !isnull(kwargs) && (ro = PyDict_GetItemString(kwargs, k)) != PyPtr() + return ro + else + return PyPtr() + end + +""" + PyArg_GetArg(T, funcname, [args, i], [kwargs, k], [default]) + +Attempt to find and convert the specified argument to a `T`. Return 0 on success, in which case the result can be retrieved with `takeresult(T)`. Return -1 on failure, with a Python error set. + +- `funcname::String` is the name of the function this is used in, for constructing error messages. +- `args::PyPtr` is a tuple of arguments, and `i::Int` an index. +- `kwargs::PyPtr` is a dict of keyword arguments, and `k::String` a key. +- `default` specifies a default value if the argument is not found. NOTE: It need not be a `T`, so when retrieving the result use `takeresult(Union{T,typeof(default)})`. +""" +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, kwargs::PyPtr, k::Union{String,Nothing}, d=NODEFAULT()) where {T} = begin + ro = PyArg_Find(args, kwargs, i, k) + if isnull(ro) + if d !== NODEFAULT() + putresult(d) + return 0 + elseif k !== nothing + PyErr_SetString(PyExc_TypeError(), "$name() did not get required argument '$k'") + return -1 + elseif i !== nothing && i ≥ 0 + PyErr_SetString(PyExc_TypeError(), "$name() takes at least $(i+1) arguments (got $(isnull(args) ? 0 : PyTuple_Size(args)))") + return -1 + else + error("impossible to satisfy this argument") + end + end + r = PyObject_TryConvert(ro, T) + if r == -1 + return -1 + elseif r == 0 + PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'") + return -1 + else + return 0 + end +end +PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Int, d=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, args, i, PyPtr(), nothing, d) +PyArg_GetArg(::Type{T}, name::String, kwargs::PyPtr, k::String, d=NODEFAULT()) where {T} = + PyArg_GetArg(T, name, PyPtr(), nothing, kwargs, k, d) diff --git a/src/cpython/bool.jl b/src/cpython/bool.jl new file mode 100644 index 00000000..d4952e77 --- /dev/null +++ b/src/cpython/bool.jl @@ -0,0 +1,22 @@ +const PyBool_Type__ref = Ref(PyPtr()) +PyBool_Type() = pyglobal(PyBool_Type__ref, :PyBool_Type) + +const Py_True__ref = Ref(PyPtr()) +Py_True() = pyglobal(Py_True__ref, :_Py_TrueStruct) + +const Py_False__ref = Ref(PyPtr()) +Py_False() = pyglobal(Py_False__ref, :_Py_FalseStruct) + +PyBool_From(x::Bool) = (o = x ? Py_True() : Py_False(); Py_IncRef(o); o) + +PyBool_Check(o) = Py_TypeCheckExact(o, PyBool_Type()) + +PyBool_TryConvertRule_bool(o, ::Type{Bool}) = + if Py_Is(o, Py_True()) + putresult(true) + elseif Py_Is(o, Py_False()) + putresult(false) + else + PyErr_SetString(PyExc_TypeError(), "Expecting a 'bool' but got a '$(PyType_Name(Py_Type(o)))'") + -1 + end diff --git a/src/cpython/buffer.jl b/src/cpython/buffer.jl new file mode 100644 index 00000000..8e7c94d9 --- /dev/null +++ b/src/cpython/buffer.jl @@ -0,0 +1,28 @@ +PyType_CheckBuffer(t) = begin + p = UnsafePtr{PyTypeObject}(t).as_buffer[] + !isnull(p) && !isnull(p.get[!]) +end + +PyObject_CheckBuffer(o) = PyType_CheckBuffer(Py_Type(o)) + +PyObject_GetBuffer(o, b, flags) = begin + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if isnull(p) || isnull(p.get[]) + PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'") + return Cint(-1) + end + ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) +end + +PyBuffer_Release(_b) = begin + b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) + o = b.obj[] + isnull(o) && return + p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] + if (!isnull(p) && !isnull(p.release[])) + ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) + end + b.obj[] = C_NULL + Py_DecRef(o) + return +end diff --git a/src/cpython/bytes.jl b/src/cpython/bytes.jl new file mode 100644 index 00000000..bbbe3ffb --- /dev/null +++ b/src/cpython/bytes.jl @@ -0,0 +1,40 @@ +@cdef :PyBytes_FromStringAndSize PyPtr (Ptr{Cchar}, Py_ssize_t) +@cdef :PyBytes_AsStringAndSize Cint (PyPtr, Ptr{Ptr{Cchar}}, Ptr{Py_ssize_t}) + +const PyBytes_Type__ref = Ref(PyPtr()) +PyBytes_Type() = pyglobal(PyBytes_Type__ref, :PyBytes_Type) + +PyBytes_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_BYTES_SUBCLASS) +PyBytes_CheckExact(o) = Py_TypeCheckExact(o, PyBytes_Type()) + +PyBytes_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyBytes_FromStringAndSize(pointer(s), sizeof(s)) + +PyBytes_AsString(o) = begin + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return "" + Base.unsafe_string(ptr[], len[]) +end + +PyBytes_AsVector(o, ::Type{T}=UInt8) where {T} = begin + T in (Int8, UInt8) || throw(MethodError(PyBytes_AsVector, (o, T))) + ptr = Ref{Ptr{Cchar}}() + len = Ref{Py_ssize_t}() + err = PyBytes_AsStringAndSize(o, ptr, len) + ism1(err) && return T[] + copy(Base.unsafe_wrap(Vector{T}, Ptr{T}(ptr[]), len[])) +end + +PyBytes_TryConvertRule_vector(o, ::Type{Vector{X}}) where {X} = begin + v = PyBytes_AsVector(o, X) + isempty(v) && PyErr_IsSet() && return -1 + return putresult(v) +end + +PyBytes_TryConvertRule_string(o, ::Type{String}) = begin + v = PyBytes_AsString(o) + isempty(v) && PyErr_IsSet() && return -1 + return putresult(v) +end diff --git a/src/cpython/collections.jl b/src/cpython/collections.jl new file mode 100644 index 00000000..e416da52 --- /dev/null +++ b/src/cpython/collections.jl @@ -0,0 +1,218 @@ +PyABC_Register(s, t) = begin + r = PyObject_GetAttrString(t, "register") + isnull(r) && return Cint(-1) + u = PyObject_CallNice(r, PyObjectRef(s)) + Py_DecRef(r) + isnull(u) && return Cint(-1) + Py_DecRef(u) + Cint(0) +end + +for n in [:Container, :Hashable, :Iterable, :Iterator, :Reversible, :Generator, :Sized, + :Callable, :Collection, :Sequence, :MutableSequence, :ByteString, :Set, :MutableSet, + :Mapping, :MutableMapping, :MappingView, :ItemsView, :KeysView, :ValuesView, + :Awaitable, :Coroutine, :AsyncIterable, :AsyncIterator, :AsyncGenerator] + p = Symbol(:Py, n, :ABC) + t = Symbol(p, :_Type) + tr = Symbol(p, :__ref) + c = Symbol(p, :_Check) + @eval const $tr = Ref(PyPtr()) + @eval $t(doimport::Bool=true) = begin + ptr = $tr[] + isnull(ptr) || return ptr + a = doimport ? PyImport_ImportModule("collections.abc") : PyImport_GetModule("collections.abc") + isnull(a) && return a + b = PyObject_GetAttrString(a, $(string(n))) + Py_DecRef(a) + isnull(b) && return b + $tr[] = b + end + @eval $c(o) = begin + t = $t(false) + isnull(t) && return (PyErr_IsSet() ? Cint(-1) : Cint(0)) + PyObject_IsInstance(o, t) + end +end + +PyIterable_ConvertRule_vector(o, ::Type{S}) where {S<:Vector} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + xo = PyIter_Next(it) + if !isnull(xo) + r = PyObject_TryConvert(xo, eltype(xs)) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(eltype(xs)) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(xs) + end + end +end +PyIterable_ConvertRule_vector(o, ::Type{Vector}) = + PyIterable_ConvertRule_vector(o, Vector{Python.PyObject}) + +PyIterable_ConvertRule_set(o, ::Type{S}) where {S<:Set} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + xo = PyIter_Next(it) + if !isnull(xo) + r = PyObject_TryConvert(xo, eltype(xs)) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(eltype(xs)) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(xs) + end + end +end +PyIterable_ConvertRule_set(o, ::Type{Set}) = + PyIterable_ConvertRule_set(o, T, Set{Python.PyObject}) + +PyIterable_ConvertRule_tuple(o, ::Type{S}) where {S<:Tuple} = begin + if !(Tuple isa DataType) + PyErr_SetString(PyExc_Exception(), "When converting Python 'tuple' to Julia 'Tuple', the destination type must be a 'DataType', i.e. not parametric and not a union. Got '$S'.") + return -1 + end + ts = S.parameters + if !isempty(ts) && Base.isvarargtype(ts[end]) + isvararg = true + vartype = ts[end].body.parameters[1] + ts = ts[1:end-1] + else + isvararg = false + end + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = Union{ts..., isvararg ? vartype : Union{}}[] + i = 0 + while true + xo = PyIter_Next(it) + if !isnull(xo) + i += 1 + if i ≤ length(ts) + t = ts[i] + elseif isvararg + t = vartype + else + Py_DecRef(it) + Py_DecRef(xo) + return -1 + end + r = PyObject_TryConvert(xo, t) + Py_DecRef(xo) + r == 1 || (Py_DecRef(it); return r) + x = takeresult(t) + push!(xs, x) + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(S(xs)) + end + end +end +PyIterable_ConvertRule_tuple(o, ::Type{Tuple{}}) = putresult(()) + +PyIterable_ConvertRule_pair(o, ::Type{Pair{K,V}}) where {K,V} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + # get the first item + ko = PyIter_Next(it) + if isnull(ko) + Py_DecRef(it) + return PyErr_IsSet() ? -1 : 0 + end + # convert it + r = PyObject_TryConvert(ko, K) + Py_DecRef(ko) + if r != 1 + Py_DecRef(it) + return r + end + k = takeresult(K) + # get the second item + vo = PyIter_Next(it) + if isnull(vo) + Py_DecRef(it) + return PyErr_IsSet() ? -1 : 0 + end + # convert it + r = PyObject_TryConvert(vo, V) + Py_DecRef(vo) + if r != 1 + Py_DecRef(it) + return r + end + v = takeresult(V) + # too many values? + xo = PyIter_Next(it) + if !isnull(xo) + Py_DecRef(xo) + Py_DecRef(it) + return 0 + end + # done + Py_DecRef(it) + putresult(Pair{K,V}(k, v)) +end +PyIterable_ConvertRule_pair(o, ::Type{Pair{K}}) where {K} = + PyIterable_ConvertRule_pair(o, Pair{K,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{Pair{K,V} where K}) where {V} = + PyIterable_ConvertRule_pair(o, Pair{Python.PyObject,V}) +PyIterable_ConvertRule_pair(o, ::Type{Pair}) = + PyIterable_ConvertRule_pair(o, Pair{Python.PyObject,Python.PyObject}) +PyIterable_ConvertRule_pair(o, ::Type{S}) where {S<:Pair} = begin + PyErr_SetString(PyExc_Exception(), "When converting Python iterable to Julia 'Pair', the destination type cannot be too complicated: the two types must either be fully specified or left unspecified. Got '$S'.") + return -1 +end + +PyMapping_ConvertRule_dict(o, ::Type{S}) where {S<:Dict} = begin + it = PyObject_GetIter(o) + isnull(it) && return -1 + xs = S() + while true + ko = PyIter_Next(it) + if !isnull(ko) + # get the key + r = PyObject_TryConvert(ko, keytype(xs)) + r == 1 || (Py_DecRef(it); Py_DecRef(ko); return r) + k = takeresult(keytype(xs)) + # get the value + vo = PyObject_GetItem(o, ko) + isnull(vo) && (Py_DecRef(it); Py_DecRef(ko); return -1) + r = PyObject_TryConvert(vo, valtype(xs)) + Py_DecRef(vo) + Py_DecRef(ko) + r == 1 || (Py_DecRef(it); return -1) + v = takeresult(valtype(xs)) + xs[k] = v + elseif PyErr_IsSet() + Py_DecRef(it) + return -1 + else + Py_DecRef(it) + return putresult(xs) + end + end +end +PyMapping_ConvertRule_dict(o, ::Type{Dict{K}}) where {K} = + PyMapping_ConvertRule_dict(o, Dict{K,Python.PyObject}) +PyMapping_ConvertRule_dict(o, ::Type{Dict{K,V} where K}) where {V} = + PyMapping_ConvertRule_dict(o, Dict{Python.PyObject,V}) +PyMapping_ConvertRule_dict(o, ::Type{Dict}) = + PyMapping_ConvertRule_dict(o, Dict{Python.PyObject,Python.PyObject}) diff --git a/src/cpython/complex.jl b/src/cpython/complex.jl new file mode 100644 index 00000000..9d3777d8 --- /dev/null +++ b/src/cpython/complex.jl @@ -0,0 +1,32 @@ +@cdef :PyComplex_FromDoubles PyPtr (Cdouble, Cdouble) +@cdef :PyComplex_RealAsDouble Cdouble (PyPtr,) +@cdef :PyComplex_ImagAsDouble Cdouble (PyPtr,) +@cdef :PyComplex_AsCComplex Py_complex (PyPtr,) + +const PyComplex_Type__ref = Ref(PyPtr()) +PyComplex_Type() = pyglobal(PyComplex_Type__ref, :PyComplex_Type) + +PyComplex_Check(o) = Py_TypeCheck(o, PyComplex_Type()) + +PyComplex_CheckExact(o) = Py_TypeCheckExact(o, PyComplex_Type()) + +PyComplex_AsComplex(o) = begin + r = PyComplex_AsCComplex(o) + Complex(r.real, r.imag) +end + +PyComplex_From(x::Union{Float16,Float32,Float64}) = PyComplex_FromDoubles(x, 0) +PyComplex_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_FromDoubles(real(x), imag(x)) + +# "Complexable" means a 'complex' or anything with a '__complex__' method +PyComplexable_TryConvertRule_convert(o, ::Type{S}) where {S} = begin + x = PyComplex_AsComplex(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(convert(S, x)) +end + +PyComplexable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin + x = PyComplexable_AsComplex(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(tryconvert(S, x)) +end diff --git a/src/cpython/consts.jl b/src/cpython/consts.jl new file mode 100644 index 00000000..6401c3c0 --- /dev/null +++ b/src/cpython/consts.jl @@ -0,0 +1,322 @@ +@enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1 + +const Py_single_input = 256 +const Py_file_input = 257 +const Py_eval_input = 258 +const Py_func_type_input = 345 + +const Py_LT = Cint(0) +const Py_LE = Cint(1) +const Py_EQ = Cint(2) +const Py_NE = Cint(3) +const Py_GT = Cint(4) +const Py_GE = Cint(5) + +const Py_METH_VARARGS = 0x0001 # args are a tuple of arguments +const Py_METH_KEYWORDS = 0x0002 # two arguments: the varargs and the kwargs +const Py_METH_NOARGS = 0x0004 # no arguments (NULL argument pointer) +const Py_METH_O = 0x0008 # single argument (not wrapped in tuple) +const Py_METH_CLASS = 0x0010 # for class methods +const Py_METH_STATIC = 0x0020 # for static methods + +const Py_T_SHORT =0 +const Py_T_INT =1 +const Py_T_LONG =2 +const Py_T_FLOAT =3 +const Py_T_DOUBLE =4 +const Py_T_STRING =5 +const Py_T_OBJECT =6 +const Py_T_CHAR =7 +const Py_T_BYTE =8 +const Py_T_UBYTE =9 +const Py_T_USHORT =10 +const Py_T_UINT =11 +const Py_T_ULONG =12 +const Py_T_STRING_INPLACE =13 +const Py_T_BOOL =14 +const Py_T_OBJECT_EX =16 +const Py_T_LONGLONG =17 # added in Python 2.5 +const Py_T_ULONGLONG =18 # added in Python 2.5 +const Py_T_PYSSIZET =19 # added in Python 2.6 +const Py_T_NONE =20 # added in Python 3.0 + +const Py_READONLY = 1 +const Py_READ_RESTRICTED = 2 +const Py_WRITE_RESTRICTED = 4 +const Py_RESTRICTED = (Py_READ_RESTRICTED | Py_WRITE_RESTRICTED) + +const PyBUF_MAX_NDIM = 64 + +# Flags for getting buffers +const PyBUF_SIMPLE = 0x0 +const PyBUF_WRITABLE = 0x0001 +const PyBUF_WRITEABLE = PyBUF_WRITABLE +const PyBUF_FORMAT = 0x0004 +const PyBUF_ND = 0x0008 +const PyBUF_STRIDES = (0x0010 | PyBUF_ND) +const PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES) +const PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES) +const PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES) +const PyBUF_INDIRECT = (0x0100 | PyBUF_STRIDES) + +const PyBUF_CONTIG = (PyBUF_ND | PyBUF_WRITABLE) +const PyBUF_CONTIG_RO = (PyBUF_ND) + +const PyBUF_STRIDED = (PyBUF_STRIDES | PyBUF_WRITABLE) +const PyBUF_STRIDED_RO = (PyBUF_STRIDES) + +const PyBUF_RECORDS = (PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_RECORDS_RO = (PyBUF_STRIDES | PyBUF_FORMAT) + +const PyBUF_FULL = (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_FULL_RO = (PyBUF_INDIRECT | PyBUF_FORMAT) + +const PyBUF_READ = 0x100 +const PyBUF_WRITE = 0x200 + +# Python 2.7 +const Py_TPFLAGS_HAVE_GETCHARBUFFER = (0x00000001<<0) +const Py_TPFLAGS_HAVE_SEQUENCE_IN = (0x00000001<<1) +const Py_TPFLAGS_GC = 0 # was sometimes (0x00000001<<2) in Python <= 2.1 +const Py_TPFLAGS_HAVE_INPLACEOPS = (0x00000001<<3) +const Py_TPFLAGS_CHECKTYPES = (0x00000001<<4) +const Py_TPFLAGS_HAVE_RICHCOMPARE = (0x00000001<<5) +const Py_TPFLAGS_HAVE_WEAKREFS = (0x00000001<<6) +const Py_TPFLAGS_HAVE_ITER = (0x00000001<<7) +const Py_TPFLAGS_HAVE_CLASS = (0x00000001<<8) +const Py_TPFLAGS_HAVE_INDEX = (0x00000001<<17) +const Py_TPFLAGS_HAVE_NEWBUFFER = (0x00000001<<21) +const Py_TPFLAGS_STRING_SUBCLASS = (0x00000001<<27) + +# Python 3.0+ has only these: +const Py_TPFLAGS_HEAPTYPE = (0x00000001<<9) +const Py_TPFLAGS_BASETYPE = (0x00000001<<10) +const Py_TPFLAGS_READY = (0x00000001<<12) +const Py_TPFLAGS_READYING = (0x00000001<<13) +const Py_TPFLAGS_HAVE_GC = (0x00000001<<14) +const Py_TPFLAGS_HAVE_VERSION_TAG = (0x00000001<<18) +const Py_TPFLAGS_VALID_VERSION_TAG = (0x00000001<<19) +const Py_TPFLAGS_IS_ABSTRACT = (0x00000001<<20) +const Py_TPFLAGS_INT_SUBCLASS = (0x00000001<<23) +const Py_TPFLAGS_LONG_SUBCLASS = (0x00000001<<24) +const Py_TPFLAGS_LIST_SUBCLASS = (0x00000001<<25) +const Py_TPFLAGS_TUPLE_SUBCLASS = (0x00000001<<26) +const Py_TPFLAGS_BYTES_SUBCLASS = (0x00000001<<27) +const Py_TPFLAGS_UNICODE_SUBCLASS = (0x00000001<<28) +const Py_TPFLAGS_DICT_SUBCLASS = (0x00000001<<29) +const Py_TPFLAGS_BASE_EXC_SUBCLASS = (0x00000001<<30) +const Py_TPFLAGS_TYPE_SUBCLASS = (0x00000001<<31) + +# only use this if we have the stackless extension +const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION = (0x00000003<<15) + +const Py_hash_t = Cssize_t +const Py_ssize_t = Cssize_t + +@kwdef struct Py_complex + real :: Cdouble = 0.0 + imag :: Cdouble = 0.0 +end + +@kwdef struct PyObject + # assumes _PyObject_HEAD_EXTRA is empty + refcnt :: Py_ssize_t = 0 + type :: Ptr{PyObject} = C_NULL +end + +const PyPtr = Ptr{PyObject} + +struct PyObjectRef + ptr :: PyPtr +end +ispyreftype(::Type{PyObjectRef}) = true +pyptr(o::PyObjectRef) = o.ptr +Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr + +@kwdef struct PyVarObject + ob_base :: PyObject = PyObject() + size :: Py_ssize_t = 0 +end + +@kwdef struct PyMethodDef + name :: Cstring = C_NULL + meth :: Ptr{Cvoid} = C_NULL + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyGetSetDef + name :: Cstring = C_NULL + get :: Ptr{Cvoid} = C_NULL + set :: Ptr{Cvoid} = C_NULL + doc :: Cstring = C_NULL + closure :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyMemberDef + name :: Cstring = C_NULL + typ :: Cint = 0 + offset :: Py_ssize_t = 0 + flags :: Cint = 0 + doc :: Cstring = C_NULL +end + +@kwdef struct PyNumberMethods + add :: Ptr{Cvoid} = C_NULL # (o,o)->o + subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + divmod :: Ptr{Cvoid} = C_NULL # (o,o)->o + power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + negative :: Ptr{Cvoid} = C_NULL # (o)->o + positive :: Ptr{Cvoid} = C_NULL # (o)->o + absolute :: Ptr{Cvoid} = C_NULL # (o)->o + bool :: Ptr{Cvoid} = C_NULL # (o)->Cint + invert :: Ptr{Cvoid} = C_NULL # (o)->o + lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + and :: Ptr{Cvoid} = C_NULL # (o,o)->o + xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + or :: Ptr{Cvoid} = C_NULL # (o,o)->o + int :: Ptr{Cvoid} = C_NULL # (o)->o + _reserved :: Ptr{Cvoid} = C_NULL + float :: Ptr{Cvoid} = C_NULL # (o)->o + inplace_add :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_subtract :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_multiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_remainder :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_power :: Ptr{Cvoid} = C_NULL # (o,o,o)->o + inplace_lshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_rshift :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_and :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_xor :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_or :: Ptr{Cvoid} = C_NULL # (o,o)->o + floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_floordivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_truedivide :: Ptr{Cvoid} = C_NULL # (o,o)->o + index :: Ptr{Cvoid} = C_NULL # (o)->o + matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_matrixmultiply :: Ptr{Cvoid} = C_NULL # (o,o)->o +end + +@kwdef struct PySequenceMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o + _was_item :: Ptr{Cvoid} = C_NULL + ass_item :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t,o)->Cint + _was_ass_slice :: Ptr{Cvoid} = C_NULL + contains :: Ptr{Cvoid} = C_NULL # (o,o)->Cint + inplace_concat :: Ptr{Cvoid} = C_NULL # (o,o)->o + inplace_repeat :: Ptr{Cvoid} = C_NULL # (o,Py_ssize_t)->o +end + +@kwdef struct PyMappingMethods + length :: Ptr{Cvoid} = C_NULL # (o)->Py_ssize_t + subscript :: Ptr{Cvoid} = C_NULL # (o,o)->o + ass_subscript :: Ptr{Cvoid} = C_NULL # (o,o,o)->Cint +end + +@kwdef struct PyBufferProcs + get :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}, Cint) -> Cint + release :: Ptr{Cvoid} = C_NULL # (o, Ptr{Py_buffer}) -> Cvoid +end + +@kwdef struct Py_buffer + buf :: Ptr{Cvoid} = C_NULL + obj :: Ptr{Cvoid} = C_NULL + len :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + readonly :: Cint = 0 + ndim :: Cint = 0 + format :: Cstring = C_NULL + shape :: Ptr{Py_ssize_t} = C_NULL + strides :: Ptr{Py_ssize_t} = C_NULL + suboffsets :: Ptr{Py_ssize_t} = C_NULL + internal :: Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyTypeObject + ob_base :: PyVarObject = PyVarObject() + name :: Cstring = C_NULL + + basicsize :: Py_ssize_t = 0 + itemsize :: Py_ssize_t = 0 + + dealloc :: Ptr{Cvoid} = C_NULL + vectorcall_offset :: Py_ssize_t = 0 + getattr :: Ptr{Cvoid} = C_NULL + setattr :: Ptr{Cvoid} = C_NULL + as_async :: Ptr{Cvoid} = C_NULL + repr :: Ptr{Cvoid} = C_NULL + + as_number :: Ptr{PyNumberMethods} = C_NULL + as_sequence :: Ptr{PySequenceMethods} = C_NULL + as_mapping :: Ptr{PyMappingMethods} = C_NULL + + hash :: Ptr{Cvoid} = C_NULL + call :: Ptr{Cvoid} = C_NULL + str :: Ptr{Cvoid} = C_NULL + getattro :: Ptr{Cvoid} = C_NULL + setattro :: Ptr{Cvoid} = C_NULL + + as_buffer :: Ptr{PyBufferProcs} = C_NULL + + flags :: Culong = 0 + + doc :: Cstring = C_NULL + + traverse :: Ptr{Cvoid} = C_NULL + + clear :: Ptr{Cvoid} = C_NULL + + richcompare :: Ptr{Cvoid} = C_NULL + + weaklistoffset :: Py_ssize_t = 0 + + iter :: Ptr{Cvoid} = C_NULL + iternext :: Ptr{Cvoid} = C_NULL + + methods :: Ptr{PyMethodDef} = C_NULL + members :: Ptr{PyMemberDef} = C_NULL + getset :: Ptr{PyGetSetDef} = C_NULL + base :: PyPtr = C_NULL + dict :: PyPtr = C_NULL + descr_get :: Ptr{Cvoid} = C_NULL + descr_set :: Ptr{Cvoid} = C_NULL + dictoffset :: Py_ssize_t = 0 + init :: Ptr{Cvoid} = C_NULL + alloc :: Ptr{Cvoid} = C_NULL + new :: Ptr{Cvoid} = C_NULL + free :: Ptr{Cvoid} = C_NULL + is_gc :: Ptr{Cvoid} = C_NULL + bases :: PyPtr = C_NULL + mro :: PyPtr = C_NULL + cache :: PyPtr = C_NULL + subclasses :: PyPtr = C_NULL + weaklist :: PyPtr = C_NULL + del :: Ptr{Cvoid} = C_NULL + + version_tag :: Cuint = 0 + + finalize :: Ptr{Cvoid} = C_NULL + vectorcall :: Ptr{Cvoid} = C_NULL + +end + +const PyTypePtr = Ptr{PyTypeObject} + +@kwdef struct PySimpleObject{T} + ob_base :: PyObject = PyObject() + value :: T +end + +PySimpleObject_GetValue(__o, ::Type{T}) where {T} = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + UnsafePtr{PySimpleObject{T}}(o).value[!] :: T + end +end diff --git a/src/cpython/ctypes.jl b/src/cpython/ctypes.jl new file mode 100644 index 00000000..ffde609d --- /dev/null +++ b/src/cpython/ctypes.jl @@ -0,0 +1,7 @@ +struct PySimpleCData_TryConvert_value{R,tr} end + +(::PySimpleCData_TryConvert_value{R,tr})(o, ::Type{S}) where {S,R,tr} = begin + ptr = PySimpleObject_GetValue(o, Ptr{R}) + val = unsafe_load(ptr) + putresult(tr ? tryconvert(S, val) : convert(S, val)) +end diff --git a/src/cpython/dict.jl b/src/cpython/dict.jl new file mode 100644 index 00000000..3cee4a00 --- /dev/null +++ b/src/cpython/dict.jl @@ -0,0 +1,62 @@ +@cdef :PyDict_New PyPtr () +@cdef :PyDict_GetItem PyPtr (PyPtr, PyPtr) +@cdef :PyDict_GetItemString PyPtr (PyPtr, Cstring) +@cdef :PyDict_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyDict_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyDict_DelItemString Cint (PyPtr, Cstring) + +const PyDict_Type__ref = Ref(PyPtr()) +PyDict_Type() = pyglobal(PyDict_Type__ref, :PyDict_Type) + +PyDict_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_DICT_SUBCLASS) + +PyDict_CheckExact(o) = Py_TypeCheckExact(o, PyDict_Type()) + +PyDict_FromPairs(kvs) = begin + r = PyDict_New() + isnull(r) && return PyPtr() + try + for (k,v) in kvs + ko = PyObject_From(k) + isnull(ko) && (Py_DecRef(r); return PyPtr()) + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); Py_DecRef(ko); return PyPtr()) + err = PyDict_SetItem(r, ko, vo) + Py_DecRef(ko) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end + +PyDict_FromStringPairs(kvs) = begin + r = PyDict_New() + isnull(r) && return PyPtr() + try + for (k,v) in kvs + vo = PyObject_From(v) + isnull(vo) && (Py_DecRef(r); return PyPtr()) + err = PyDict_SetItemString(r, string(k), vo) + Py_DecRef(vo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end + +PyDict_From(x::AbstractDict) = PyDict_FromPairs(x) +PyDict_From(x::AbstractDict{String}) = PyDict_FromStringPairs(x) +PyDict_From(x::AbstractDict{Symbol}) = PyDict_FromStringPairs(x) +PyDict_From(x::NamedTuple) = PyDict_FromStringPairs(pairs(x)) +PyDict_From(x::Base.Iterators.Pairs) = PyDict_FromPairs(x) +PyDict_From(x::Base.Iterators.Pairs{String}) = PyDict_FromStringPairs(x) +PyDict_From(x::Base.Iterators.Pairs{Symbol}) = PyDict_FromStringPairs(x) diff --git a/src/cpython/float.jl b/src/cpython/float.jl new file mode 100644 index 00000000..37e0315f --- /dev/null +++ b/src/cpython/float.jl @@ -0,0 +1,24 @@ +@cdef :PyFloat_FromDouble PyPtr (Cdouble,) +@cdef :PyFloat_AsDouble Cdouble (PyPtr,) + +const PyFloat_Type__ref = Ref(PyPtr()) +PyFloat_Type() = pyglobal(PyFloat_Type__ref, :PyFloat_Type) + +PyFloat_Check(o) = Py_TypeCheck(o, PyFloat_Type()) + +PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type()) + +PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o) + +# "Floatable" means a 'float' or anything with a '__float__' method +PyFloatable_TryConvertRule_convert(o, ::Type{S}) where {S} = begin + x = PyFloat_AsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(convert(S, x)) +end + +PyFloatable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin + x = PyFloat_AsDouble(o) + ism1(x) && PyErr_IsSet() && return -1 + putresult(tryconvert(S, x)) +end diff --git a/src/cpython/fraction.jl b/src/cpython/fraction.jl new file mode 100644 index 00000000..469ec99d --- /dev/null +++ b/src/cpython/fraction.jl @@ -0,0 +1,31 @@ +const PyFraction_Type__ref = Ref(PyPtr()) +PyFraction_Type() = begin + ptr = PyFraction_Type__ref[] + if isnull(ptr) + m = PyImport_ImportModule("fractions") + isnull(m) && return ptr + ptr = PyObject_GetAttrString(m, "Fraction") + Py_DecRef(m) + isnull(m) && return ptr + PyFraction_Type__ref[] = ptr + end + ptr +end + +PyFraction_From(x::Union{Rational,Integer}) = begin + t = PyFraction_Type() + isnull(t) && return PyPtr() + a = PyTuple_New(2) + isnull(a) && return PyPtr() + b = PyLong_From(numerator(x)) + isnull(b) && (Py_DecRef(a); return PyPtr()) + err = PyTuple_SetItem(a, 0, b) + ism1(err) && (Py_DecRef(a); return PyPtr()) + b = PyLong_From(denominator(x)) + isnull(b) && (Py_DecRef(a); return PyPtr()) + err = PyTuple_SetItem(a, 1, b) + ism1(err) && (Py_DecRef(a); return PyPtr()) + r = PyObject_CallObject(t, a) + Py_DecRef(a) + return r +end diff --git a/src/cpython/fundamentals.jl b/src/cpython/fundamentals.jl new file mode 100644 index 00000000..7c48d29b --- /dev/null +++ b/src/cpython/fundamentals.jl @@ -0,0 +1,104 @@ +### INITIALIZE + +@cdef :Py_Initialize Cvoid () +@cdef :Py_InitializeEx Cvoid (Cint,) +@cdef :Py_Finalize Cvoid () +@cdef :Py_FinalizeEx Cint () +@cdef :Py_AtExit Cint (Ptr{Cvoid},) +@cdef :Py_IsInitialized Cint () + +@cdef :Py_SetPythonHome Cvoid (Cwstring,) +@cdef :Py_SetProgramName Cvoid (Cwstring,) +@cdef :Py_GetVersion Cstring () + +### REFCOUNT + +@cdef :Py_IncRef Cvoid (PyPtr,) +@cdef :Py_DecRef Cvoid (PyPtr,) +Py_RefCnt(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).refcnt[] + +Py_DecRef(f::Function, o::Ptr, dflt=PYERR()) = + isnull(o) ? dflt : (r=f(o); Py_DecRef(o); r) + +Py_Is(o1, o2) = Base.unsafe_convert(PyPtr, o1) == Base.unsafe_convert(PyPtr, o2) + +### EVAL + +@cdef :PyEval_EvalCode PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :Py_CompileString PyPtr (Cstring, Cstring, Cint) +@cdef :PyEval_GetBuiltins PyPtr () + +### GIL & THREADS + +@cdef :PyEval_SaveThread Ptr{Cvoid} () +@cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},) +@cdef :PyGILState_Ensure PyGILState_STATE () +@cdef :PyGILState_Release Cvoid (PyGILState_STATE,) + +### IMPORT + +@cdef :PyImport_ImportModule PyPtr (Cstring,) +@cdef :PyImport_Import PyPtr (PyPtr,) +@cdef :PyImport_GetModuleDict PyPtr () + +PyImport_GetModule(name) = begin + ms = PyImport_GetModuleDict() + ok = PyMapping_HasKeyString(ms, name) + ism1(ok) && return PyPtr() + ok != 0 ? PyMapping_GetItemString(ms, name) : PyPtr() +end + +### ERRORS + +@cdef :PyErr_Occurred PyPtr () +@cdef :PyErr_GivenExceptionMatches Cint (PyPtr, PyPtr) +@cdef :PyErr_Clear Cvoid () +@cdef :PyErr_SetNone Cvoid (PyPtr,) +@cdef :PyErr_SetString Cvoid (PyPtr, Cstring) +@cdef :PyErr_SetObject Cvoid (PyPtr, PyPtr) +@cdef :PyErr_Fetch Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_NormalizeException Cvoid (Ptr{PyPtr}, Ptr{PyPtr}, Ptr{PyPtr}) +@cdef :PyErr_Restore Cvoid (PyPtr, PyPtr, PyPtr) + +PyErr_IsSet() = !isnull(PyErr_Occurred()) + +PyErr_IsSet(t) = (o=PyErr_Occurred(); !isnull(o) && PyErr_GivenExceptionMatches(o,t)!=0) + +function PyErr_FetchTuple(normalize::Bool=false) + t = Ref{PyPtr}() + v = Ref{PyPtr}() + b = Ref{PyPtr}() + PyErr_Fetch(t, v, b) + normalize && PyErr_NormalizeException(t, v, b) + (t[], v[], b[]) +end + +### EXCEPTIONS + +for x in [:BaseException, :Exception, :StopIteration, :GeneratorExit, :ArithmeticError, + :LookupError, :AssertionError, :AttributeError, :BufferError, :EOFError, + :FloatingPointError, :OSError, :ImportError, :IndexError, :KeyError, :KeyboardInterrupt, + :MemoryError, :NameError, :OverflowError, :RuntimeError, :RecursionError, + :NotImplementedError, :SyntaxError, :IndentationError, :TabError, :ReferenceError, + :SystemError, :SystemExit, :TypeError, :UnboundLocalError, :UnicodeError, + :UnicodeEncodeError, :UnicodeDecodeError, :UnicodeTranslateError, :ValueError, + :ZeroDivisionError, :BlockingIOError, :BrokenPipeError, :ChildProcessError, + :ConnectionError, :ConnectionAbortedError, :ConnectionRefusedError, :FileExistsError, + :FileNotFoundError, :InterruptedError, :IsADirectoryError, :NotADirectoryError, + :PermissionError, :ProcessLookupError, :TimeoutError, :EnvironmentError, :IOError, + :WindowsError, :Warning, :UserWarning, :DeprecationWarning, :PendingDeprecationWarning, + :SyntaxWarning, :RuntimeWarning, :FutureWarning, :ImportWarning, :UnicodeWarning, + :BytesWarning, :ResourceWarning] + f = Symbol(:PyExc_, x) + r = Symbol(f, :__ref) + @eval const $r = Ref(PyPtr()) + @eval $f() = pyloadglobal($r, $(QuoteNode(f))) +end + +### INPUT HOOK + +function PyOS_RunInputHook() + hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook))) + isnull(hook) || ccall(hook, Cint, ()) + nothing +end diff --git a/src/cpython/int.jl b/src/cpython/int.jl new file mode 100644 index 00000000..abb65931 --- /dev/null +++ b/src/cpython/int.jl @@ -0,0 +1,63 @@ +@cdef :PyLong_FromLongLong PyPtr (Clonglong,) +@cdef :PyLong_FromUnsignedLongLong PyPtr (Culonglong,) +@cdef :PyLong_FromString PyPtr (Cstring, Ptr{Cvoid}, Cint) +@cdef :PyLong_AsLongLong Clonglong (PyPtr,) +@cdef :PyLong_AsUnsignedLongLong Culonglong (PyPtr,) + +const PyLong_Type__ref = Ref(PyPtr()) +PyLong_Type() = pyglobal(PyLong_Type__ref, :PyLong_Type) + +PyLong_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LONG_SUBCLASS) + +PyLong_CheckExact(o) = Py_TypeCheckExact(o, PyLong_Type()) + +PyLong_From(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128,BigInt}) = + if x isa Signed && typemin(Clonglong) ≤ x ≤ typemax(Clonglong) + PyLong_FromLongLong(x) + elseif typemin(Culonglong) ≤ x ≤ typemax(Culonglong) + PyLong_FromUnsignedLongLong(x) + else + PyLong_FromString(string(x), C_NULL, 10) + end + +PyLong_From(x::Integer) = begin + y = tryconvert(BigInt, x) + y === PYERR() && return PyPtr() + y === NOTIMPLEMENTED() && (PyErr_SetString(PyExc_NotImplementedError(), "Cannot convert this Julia '$(typeof(x))' to a Python 'int'"); return PyPtr()) + PyLong_From(y::BigInt) +end + +# "Longable" means an 'int' or anything with an '__int__' method. +PyLongable_TryConvertRule_integer(o, ::Type{S}) where {S<:Integer} = begin + # first try to convert to Clonglong (or Culonglong if unsigned) + x = S <: Unsigned ? PyLong_AsUnsignedLongLong(o) : PyLong_AsLongLong(o) + if !ism1(x) || !PyErr_IsSet() + # success + return putresult(tryconvert(S, x)) + elseif PyErr_IsSet(PyExc_OverflowError()) + # overflows Clonglong or Culonglong + PyErr_Clear() + if S in (Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128) && typemin(typeof(x)) ≤ typemin(S) && typemax(S) ≤ typemax(typeof(x)) + # definitely overflows S, give up now + return 0 + else + # try converting to String then BigInt then S + so = PyObject_Str(o) + isnull(so) && return -1 + s = PyUnicode_AsString(so) + Py_DecRef(so) + isempty(s) && PyErr_IsSet() && return -1 + y = tryparse(BigInt, s) + y === nothing && (PyErr_SetString(PyExc_ValueError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia 'BigInt' because its string representation cannot be parsed as an integer"); return -1) + return putresult(tryconvert(S, y)) + end + else + # other error + return -1 + end +end + +PyLongable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin + r = PyLongable_TryConvertRule_integer(o, Integer) + r == 1 ? putresult(tryconvert(S, takeresult(Integer))) : r +end diff --git a/src/cpython/iter.jl b/src/cpython/iter.jl new file mode 100644 index 00000000..84c6d74d --- /dev/null +++ b/src/cpython/iter.jl @@ -0,0 +1 @@ +@cdef :PyIter_Next PyPtr (PyPtr,) diff --git a/src/cpython/juliaany.jl b/src/cpython/juliaany.jl new file mode 100644 index 00000000..a322b82a --- /dev/null +++ b/src/cpython/juliaany.jl @@ -0,0 +1,354 @@ +const PyJuliaAnyValue_Type__ref = Ref(PyPtr()) +PyJuliaAnyValue_Type() = begin + ptr = PyJuliaAnyValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaBaseValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.AnyValue", + base = base, + repr = pyjlany_repr, + str = pyjlany_str, + getattro = pyjlany_getattro, + setattro = pyjlany_setattro, + call = pyjlany_call, + iter = pyjlany_iter, + richcompare = pyjlany_richcompare, + as_mapping = ( + length = pyjlany_length, + subscript = pyjlany_getitem, + ass_subscript = pyjlany_setitem, + ), + as_sequence = ( + contains = pyjlany_contains, + ), + methods = [ + (name="__dir__", flags=Py_METH_NOARGS, meth=pyjlany_dir), + (name="_repr_html_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/html"))), + (name="_repr_markdown_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/markdown"))), + (name="_repr_json_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/json"))), + (name="_repr_javascript_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("application/javascript"))), + (name="_repr_pdf_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("application/pdf"))), + (name="_repr_jpeg_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/jpeg"))), + (name="_repr_png_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/png"))), + (name="_repr_svg_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("image/svg+xml"))), + (name="_repr_latex_", flags=Py_METH_NOARGS, meth=pyjlany_repr_mime(MIME("text/latex"))), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaAnyValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaAnyValue_New(x) = PyJuliaValue_New(PyJuliaAnyValue_Type(), x) +PyJuliaValue_From(x) = PyJuliaAnyValue_New(x) + +pyjlany_repr(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + # s = "" + io = IOBuffer() + ioc = IOContext(io, :limit=>true, :compact=>true, :color=>true) + show(ioc, MIME("text/plain"), x) + s = String(take!(io)) + s = string("jl:", '\n' in s ? '\n' : ' ', s) + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlany_str(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = string(x) + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + return PyPtr() +end + +pyjlany_getattro(xo::PyPtr, ko::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericGetAttr(xo, ko) + if isnull(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to get the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return PyPtr() + k = pyjl_attr_py2jl(k) + try + v = getproperty(x, Symbol(k)) + PyObject_From(v) + catch err + if (err isa UndefVarError && err.var === Symbol(k)) || (err isa ErrorException && occursin("has no field", err.msg)) + PyErr_SetStringFromJuliaError(PyExc_AttributeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +propertytype(x, k) = + propertiesarefields(typeof(x)) && hasfield(typeof(x), k) ? fieldtype(typeof(x), k) : Any +@generated propertiesarefields(::Type{T}) where {T} = + which(getproperty, Tuple{T,Symbol}) == which(getproperty, Tuple{Any,Symbol}) + +pyjlany_setattro(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericSetAttr(xo, ko, vo) + if ism1(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + if isnull(vo) + PyErr_SetString(PyExc_TypeError(), "attribute deletion not supported") + return Cint(-1) + end + # Now try to set the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return Cint(-1) + k = pyjl_attr_py2jl(k) + try + V = propertytype(x, Symbol(k)) + ism1(PyObject_Convert(vo, V)) && return Cint(-1) + v = takeresult(V) + setproperty!(x, Symbol(k), v) + Cint(0) + catch err + if (err isa UndefVarError && err.var === Symbol(k)) || (err isa ErrorException && occursin("has no field", err.msg)) + PyErr_SetStringFromJuliaError(PyExc_AttributeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjl_dir(x) = propertynames(x) +pyjl_dir(x::Module) = begin + r = Symbol[] + append!(r, names(x, all=true, imported=true)) + for m in ccall(:jl_module_usings, Any, (Any,), x)::Vector + append!(r, names(m)) + end + r +end + +pyjlany_dir(xo::PyPtr, _::PyPtr) = begin + fo = PyObject_GetAttrString(PyJuliaBaseValue_Type(), "__dir__") + isnull(fo) && return PyPtr() + ro = PyObject_CallNice(fo, PyObjectRef(xo)) + Py_DecRef(fo) + isnull(ro) && return PyPtr() + x = PyJuliaValue_GetValue(xo) + ks = try + collect(map(string, pyjl_dir(x))) + catch err + Py_DecRef(ro) + PyErr_SetJuliaError(err) + return PyPtr() + end + for k in ks + ko = PyUnicode_From(pyjl_attr_jl2py(k)) + isnull(ko) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, ko) + Py_DecRef(ko) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + return ro +end + +pyjlany_call(fo::PyPtr, argso::PyPtr, kwargso::PyPtr) = begin + f = PyJuliaValue_GetValue(fo) + if isnull(argso) + args = Vector{Any}() + else + ism1(PyObject_Convert(argso, Vector{Any})) && return PyPtr() + args = takeresult(Vector{Any}) + end + if isnull(kwargso) + kwargs = Dict{Symbol, Any}() + else + ism1(PyObject_Convert(kwargso, Dict{Symbol, Any})) && return PyPtr() + kwargs = takeresult(Dict{Symbol, Any}) + end + try + x = f(args...; kwargs...) + PyObject_From(x) + catch err + if err isa MethodError && err.f === f + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +pyjlany_length(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + Py_ssize_t(length(x)) +catch err + if err isa MethodError && err.f === length + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Py_ssize_t(-1) +end + +@generated pyjl_keytype(::Type{T}) where {T} = try; keytype(T); catch; nothing; end; +pyjl_hasvarindices(::Type) = true +pyjl_hasvarindices(::Type{<:AbstractDict}) = false + +pyjl_getindices(x, ko) = + if (K = pyjl_keytype(typeof(x))) !== nothing + ism1(PyObject_Convert(ko, K)) ? PYERR() : (takeresult(K),) + elseif pyjl_hasvarindices(typeof(x)) && PyTuple_Check(ko) + ism1(PyObject_TryConvert(ko, Tuple)) ? PYERR() : takeresult(Tuple) + else + ism1(PyObject_TryConvert(ko, Any)) ? PYERR() : takeresult(Any) + end + +pyjlany_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + k = pyjl_getindices(x, ko) + k === PYERR() && return PyPtr() + try + PyObject_From(x[k...]) + catch err + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa KeyError && (err.key === k || (err.key,) === k) + PyErr_SetStringFromJuliaError(PyExc_KeyError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +@generated pyjl_valtype(::Type{T}) where {T} = try; valtype(T); catch; try eltype(T); catch; nothing; end; end; + +pyjl_getvalue(x, vo) = + if (V = pyjl_valtype(typeof(x))) !== nothing + ism1(PyObject_Convert(vo, V)) ? PYERR() : takeresult(V) + else + ism1(PyObject_Convert(vo, Any)) ? PYERR() : takeresult(Any) + end + +pyjlany_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + k = pyjl_getindices(x, ko) + k === PYERR() && return Cint(-1) + try + if isnull(vo) + delete!(x, k...) + Cint(0) + else + v = pyjl_getvalue(x, vo) + v === PYERR() && return Cint(-1) + x[k...] = v + Cint(0) + end + catch + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa KeyError && (err.key === k || (err.key,) === k) + PyErr_SetStringFromJuliaError(PyExc_KeyError(), err) + elseif err isa MethodError && err.f === delete! + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlany_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator(PyJuliaValue_GetValue(xo))) + +pyjlany_contains(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return Cint(-1) + r == 0 && return Cint(0) + v = takeresult(eltype(x)) + try + Cint(v in x) + catch err + if err isa MethodError && err.f === :in + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlany_richcompare(xo::PyPtr, yo::PyPtr, op::Cint) = begin + x = PyJuliaValue_GetValue(xo) + r = PyObject_TryConvert(yo, Any) + r == -1 && return PyPtr() + r == 0 && return PyNotImplemented_New() + y = takeresult() + try + if op == Py_EQ + PyObject_From(x == y) + elseif op == Py_NE + PyObject_From(x != y) + elseif op == Py_LE + PyObject_From(x <= y) + elseif op == Py_LT + PyObject_From(x < y) + elseif op == Py_GE + PyObject_From(x >= y) + elseif op == Py_GT + PyObject_From(x > y) + else + PyErr_SetString(PyExc_ValueError(), "bad op given to richcompare: $op") + PyPtr() + end + catch err + if err isa MethodError && err.f in (==, !=, <=, <, >=, >) + PyNotImplemented_New() + else + PyErr_SetJuliaError(err) + PyPtr() + end + end +end + +struct pyjlany_repr_mime{M<:MIME} + mime :: M +end +(f::pyjlany_repr_mime{M})(xo::PyPtr, ::PyPtr) where {M} = begin + x = PyJuliaValue_GetValue(xo) + io = IOBuffer() + try + show(io, f.mime, x) + catch err + if err isa MethodError && err.f === show && err.args === (io, f.mime, x) + return PyNone_New() + else + PyErr_SetJuliaError(err) + return PyPtr() + end + end + data = take!(io) + if istextmime(f.mime) + PyUnicode_From(data) + else + PyBytes_From(data) + end +end diff --git a/src/cpython/juliaarray.jl b/src/cpython/juliaarray.jl new file mode 100644 index 00000000..a579fb88 --- /dev/null +++ b/src/cpython/juliaarray.jl @@ -0,0 +1,391 @@ +const PyJuliaArrayValue_Type__ref = Ref(PyPtr()) +PyJuliaArrayValue_Type() = begin + ptr = PyJuliaArrayValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.ArrayValue", + base = base, + as_mapping = ( + subscript = pyjlarray_getitem, + ass_subscript = pyjlarray_setitem, + ), + as_buffer = ( + get = pyjlarray_get_buffer, + release = pyjlarray_release_buffer, + ), + getset = [ + (name="ndim", get=pyjlarray_ndim), + (name="shape", get=pyjlarray_shape), + (name="__array_interface__", get=pyjlarray_array_interface), + ], + methods = [ + (name="copy", flags=Py_METH_NOARGS, meth=pyjlarray_copy), + (name="reshape", flags=Py_METH_O, meth=pyjlarray_reshape), + (name="__array__", flags=Py_METH_NOARGS, meth=pyjlarray_array), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyCollectionABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaArrayValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaArrayValue_New(x::AbstractArray) = PyJuliaValue_New(PyJuliaArrayValue_Type(), x) +PyJuliaValue_From(x::AbstractArray) = PyJuliaArrayValue_New(x) + +pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, ko::PyPtr) = begin + if PySlice_Check(ko) + ao, co, bo = PySimpleObject_GetValue(ko, Tuple{PyPtr, PyPtr, PyPtr}) + # start + r = PyObject_TryConvert(ao, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + a = takeresult(Union{Int, Nothing}) + # step + r = PyObject_TryConvert(bo, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + b = takeresult(Union{Int, Nothing}) + # stop + r = PyObject_TryConvert(co, Union{Int, Nothing}) + r == -1 && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "slice components must be integers"); return PYERR()) + c = takeresult(Union{Int, Nothing}) + # step defaults to 1 + b′ = b === nothing ? 1 : b + if a === nothing && c === nothing + # when neither is specified, start and stop default to the full range, + # which is reversed when the step is negative + if b′ > 0 + a′ = Int(first(x)) + c′ = Int(last(x)) + elseif b′ < 0 + a′ = Int(last(x)) + c′ = Int(first(x)) + else + PyErr_SetString(PyExc_ValueError(), "step must be non-zero") + return PYERR() + end + else + # start defaults + a′ = Int(a === nothing ? first(x) : a < 0 ? (last(x) + a + 1) : (first(x) + a)) + c′ = Int(c === nothing ? last(x) : c < 0 ? (last(x) + 1 + c - sign(b′)) : (first(x) + c - sign(b′))) + end + r = a′ : b′ : c′ + if !checkbounds(Bool, x, r) + PyErr_SetString(PyExc_IndexError(), "array index out of bounds") + return PYERR() + end + r + else + r = PyObject_TryConvert(ko, Int) + ism1(r) && return PYERR() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "index must be slice or integer, got $(PyType_Name(Py_Type(ko)))"); return PYERR()) + k = takeresult(Int) + k′ = k < 0 ? (last(x) + k + 1) : (first(x) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PYERR()) + k′ + end +end + +pyjl_getarrayindices(x::AbstractArray, ko::PyPtr) = begin + kos = PyTuple_Check(ko) ? [PyTuple_GetItem(ko, i-1) for i in 1:PyTuple_Size(ko)] : [ko] + length(kos) == ndims(x) || (PyErr_SetString(PyExc_TypeError(), "expecting exactly $(ndims(x)) indices, got $(length(kos))"); return PYERR()) + ks = [] + for (i,ko) in enumerate(kos) + k = pyjl_getaxisindex(axes(x, i), ko) + k === PYERR() && return PYERR() + push!(ks, k) + end + ks +end + +pyjlarray_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + k = pyjl_getarrayindices(x, ko) + k === PYERR() && return PyPtr() + try + if all(x -> x isa Int, k) + PyObject_From(x[k...]) + else + PyObject_From(view(x, k...)) + end + catch err + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() + end +end + +pyjlarray_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + k = pyjl_getarrayindices(x, ko) + k === PYERR() && return Cint(-1) + try + if isnull(vo) + deleteat!(x, k...) + Cint(0) + else + ism1(PyObject_Convert(vo, eltype(x))) && return Cint(-1) + v = takeresult(eltype(x)) + if all(x -> x isa Int, k) + x[k...] = v + Cint(0) + else + PyErr_SetString(PyExc_TypeError(), "multiple assignment not supported") + Cint(-1) + end + end + catch + if err isa BoundsError && err.a === x + PyErr_SetStringFromJuliaError(PyExc_IndexError(), err) + elseif err isa MethodError && err.f === deleteat! + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + Cint(-1) + end +end + +pyjlarray_ndim(xo::PyPtr, ::Ptr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(ndims(x)) +end + +pyjlarray_shape(xo::PyPtr, ::Ptr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(size(x)) +end + +pyjlarray_copy(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractArray + PyObject_From(copy(x)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlarray_reshape(xo::PyPtr, arg::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractArray + r = PyObject_TryConvert(arg, Union{Int, Tuple{Vararg{Int}}}) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "shape must be an integer or tuple of integers"); return PyPtr()) + PyObject_From(reshape(x, takeresult()...)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +### Buffer Protocol + +isflagset(flags, mask) = (flags & mask) == mask + +const PYJLBUFCACHE = Dict{Ptr{Cvoid}, Any}() + +pyjl_get_buffer_impl(o::PyPtr, buf::Ptr{Py_buffer}, flags, ptr, elsz, len, ndim, fmt, sz, strds, mutable) = begin + b = UnsafePtr(buf) + c = [] + + # not influenced by flags: obj, buf, len, itemsize, ndim + b.obj[] = C_NULL + b.buf[] = ptr + b.itemsize[] = elsz + b.len[] = elsz * len + b.ndim[] = ndim + + # readonly + if isflagset(flags, PyBUF_WRITABLE) + if mutable + b.readonly[] = 1 + else + PyErr_SetString(PyExc_BufferError(), "not writable") + return Cint(-1) + end + else + b.readonly[] = mutable ? 0 : 1 + end + + # format + if isflagset(flags, PyBUF_FORMAT) + b.format[] = cacheptr!(c, fmt) + else + b.format[] = C_NULL + end + + # shape + if isflagset(flags, PyBUF_ND) + b.shape[] = cacheptr!(c, Py_ssize_t[sz...]) + else + b.shape[] = C_NULL + end + + # strides + if isflagset(flags, PyBUF_STRIDES) + b.strides[] = cacheptr!(c, Py_ssize_t[(strds .* elsz)...]) + else + if Python.size_to_cstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not C contiguous and strides not requested") + return Cint(-1) + end + b.strides[] = C_NULL + end + + # check contiguity + if isflagset(flags, PyBUF_C_CONTIGUOUS) + if Python.size_to_cstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not C contiguous") + return Cint(-1) + end + end + if isflagset(flags, PyBUF_F_CONTIGUOUS) + if Python.size_to_fstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not Fortran contiguous") + return Cint(-1) + end + end + if isflagset(flags, PyBUF_ANY_CONTIGUOUS) + if Python.size_to_cstrides(1, sz...) != strds && Python.size_to_fstrides(1, sz...) != strds + PyErr_SetString(PyExc_BufferError(), "not contiguous") + return Cint(-1) + end + end + + # suboffsets + b.suboffsets[] = C_NULL + + # internal + cptr = Base.pointer_from_objref(c) + PYJLBUFCACHE[cptr] = c + b.internal[] = cptr + + # obj + Py_IncRef(o) + b.obj[] = o + Cint(0) +end + +pyjlarray_isbufferabletype(::Type{T}) where {T} = + T in (Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}, Bool, Ptr{Cvoid}) +pyjlarray_isbufferabletype(::Type{T}) where {T<:Tuple} = + isconcretetype(T) && Base.allocatedinline(T) && all(pyjlarray_isbufferabletype, fieldtypes(T)) +pyjlarray_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = + pyjlarray_isbufferabletype(T) + +_pyjlarray_get_buffer(xo, buf, flags, x::AbstractArray) = try + if pyjlarray_isbufferabletype(eltype(x)) + pyjl_get_buffer_impl(xo, buf, flags, Base.unsafe_convert(Ptr{eltype(x)}, x), Base.aligned_sizeof(eltype(x)), length(x), ndims(x), Python.pybufferformat(eltype(x)), size(x), strides(x), Python.ismutablearray(x)) + else + error("element type is not bufferable") + end +catch err + PyErr_SetString(PyExc_BufferError(), "Buffer protocol not supported by Julia '$(typeof(x))' (details: $err)") + Cint(-1) +end + +pyjlarray_get_buffer(xo::PyPtr, buf::Ptr{Py_buffer}, flags::Cint) = + _pyjlarray_get_buffer(xo, buf, flags, PyJuliaValue_GetValue(xo)::AbstractArray) + +pyjlarray_release_buffer(xo::PyPtr, buf::Ptr{Py_buffer}) = begin + delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) + nothing +end + +### Array Interface + +pyjlarray_isarrayabletype(::Type{T}) where {T} = + T in (UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64, Bool, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}) +pyjlarray_isarrayabletype(::Type{T}) where {T<:Tuple} = + isconcretetype(T) && Base.allocatedinline(T) && all(pyjlarray_isarrayabletype, T.parameters) +pyjlarray_isarrayabletype(::Type{NamedTuple{names,types}}) where {names,types} = + pyjlarray_isarrayabletype(types) + +pyjlarray_array_interface(xo::PyPtr, ::Ptr{Cvoid}) = + _pyjlarray_array_interface(PyJuliaValue_GetValue(xo)::AbstractArray) + +PyDescrObject_From(fields) = begin + ro = PyList_New(0) + for (name, descr) in fields + descro = descr isa String ? PyUnicode_From(descr) : PyDescrObject_From(descr) + isnull(descro) && (Py_DecRef(ro); return PyPtr()) + fieldo = PyTuple_From((name, PyObjectRef(descro))) + Py_DecRef(descro) + isnull(fieldo) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, fieldo) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + ro +end + +_pyjlarray_array_interface(x::AbstractArray) = try + if pyjlarray_isarrayabletype(eltype(x)) + # gather information + shape = size(x) + data = (UInt(Base.unsafe_convert(Ptr{eltype(x)}, x)), !Python.ismutablearray(x)) + strides = Base.strides(x) .* Base.aligned_sizeof(eltype(x)) + version = 3 + typestr, descr = Python.pytypestrdescr(eltype(x)) + isempty(typestr) && error("invalid element type") + # make the dictionary + d = PyDict_From(Dict( + "shape" => shape, + "typestr" => typestr, + "data" => data, + "strides" => strides, + "version" => version, + )) + isnull(d) && return PyPtr() + if descr !== nothing + descro = PyDescrObject_From(descr) + err = PyDict_SetItemString(d, "descr", descro) + Py_DecRef(descro) + ism1(err) && (Py_DecRef(d); return PyPtr()) + end + d + else + error("invalid element type") + end +catch err + PyErr_SetString(PyExc_AttributeError(), "__array_interface__") + PyPtr() +end + +pyjlarray_array(xo::PyPtr, ::PyPtr) = begin + if PyObject_HasAttrString(xo, "__array_interface__") == 0 + # convert to a PyObjectArray + x = PyJuliaValue_GetValue(xo)::AbstractArray + y = try + Python.PyObjectArray(x) + catch err + PyErr_SetJuliaError(err) + return PyPtr() + end + yo = PyJuliaArrayValue_New(y) + isnull(yo) && return PyPtr() + else + # already supports the array interface + Py_IncRef(xo) + yo = xo + end + npo = PyImport_ImportModule("numpy") + isnull(npo) && (Py_DecRef(yo); return PyPtr()) + aao = PyObject_GetAttrString(npo, "array") + Py_DecRef(npo) + isnull(aao) && (Py_DecRef(yo); return PyPtr()) + ao = PyObject_CallNice(aao, PyObjectRef(yo)) + Py_DecRef(aao) + Py_DecRef(yo) + ao +end diff --git a/src/cpython/juliabase.jl b/src/cpython/juliabase.jl new file mode 100644 index 00000000..f49fcb83 --- /dev/null +++ b/src/cpython/juliabase.jl @@ -0,0 +1,104 @@ +const PYJLGCCACHE = Dict{PyPtr, Any}() + +@kwdef struct PyJuliaValueObject + ob_base :: PyObject = PyObject() + value :: Ptr{Cvoid} = C_NULL + weaklist :: PyPtr = C_NULL +end + +pyjlbase_new(t::PyPtr, ::PyPtr, ::PyPtr) = begin + o = ccall(UnsafePtr{PyTypeObject}(t).alloc[!], PyPtr, (PyPtr, Py_ssize_t), t, 0) + if !isnull(o) + PyJuliaValue_SetValue(o, nothing) + UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C_NULL + end + o +end + +pyjlbase_init(o::PyPtr, args::PyPtr, kwargs::PyPtr) = begin + ism1(PyArg_CheckNumArgsEq("__init__", args, 1)) && return Cint(-1) + ism1(PyArg_CheckNoKwargs("__init__", kwargs)) && return Cint(-1) + ism1(PyArg_GetArg(Any, "__init__", args, kwargs, 0)) && return Cint(-1) + PyJuliaValue_SetValue(o, takeresult(Any)) + Cint(0) +end + +pyjlbase_dealloc(o::PyPtr) = begin + delete!(PYJLGCCACHE, o) + isnull(UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) || PyObject_ClearWeakRefs(o) + ccall(UnsafePtr{PyTypeObject}(Py_Type(o)).free[!], Cvoid, (PyPtr,), o) + nothing +end + +const PyJuliaBaseValue_Type__ref = Ref(PyPtr()) +PyJuliaBaseValue_Type() = begin + ptr = PyJuliaBaseValue_Type__ref[] + if isnull(ptr) + c = [] + t = fill(PyType_Create(c, + name = "julia.ValueBase", + basicsize = sizeof(PyJuliaValueObject), + new = pyglobal(:PyType_GenericNew), + init = @cfunction(pyjlbase_init, Cint, (PyPtr, PyPtr, PyPtr)), + dealloc = @cfunction(pyjlbase_dealloc, Cvoid, (PyPtr,)), + flags = Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), + weaklistoffset = fieldoffset(PyJuliaValueObject, 3), + getattro = pyglobal(:PyObject_GenericGetAttr), + setattro = pyglobal(:PyObject_GenericSetAttr), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaBaseValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaValue_Check(o) = Py_TypeCheck(o, PyJuliaBaseValue_Type()) + +PyJuliaValue_GetValue(__o) = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + p = UnsafePtr{PyJuliaValueObject}(o) + Base.unsafe_pointer_to_objref(p.value[!]) + end +end + +PyJuliaValue_SetValue(__o, v) = begin + _o = Base.cconvert(PyPtr, __o) + GC.@preserve _o begin + o = Base.unsafe_convert(PyPtr, _o) + p = UnsafePtr{PyJuliaValueObject}(o) + p.value[!], PYJLGCCACHE[o] = Python.pointer_from_obj(v) + end +end + +PyJuliaValue_New(t, v) = begin + if isnull(t) + if !PyErr_IsSet() + PyErr_SetString(PyExc_Exception(), "Got NULL type with no error set") + end + return PyPtr() + end + bt = PyJuliaBaseValue_Type() + isnull(bt) && return PyPtr() + PyType_IsSubtype(t, bt) != 0 || (PyErr_SetString(PyExc_TypeError(), "Expecting a subtype of 'julia.ValueBase'"); return PyPtr()) + o = _PyObject_New(t) + isnull(o) && return PyPtr() + UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C_NULL + PyJuliaValue_SetValue(o, v) + o +end + +PyJuliaBaseValue_New(v) = begin + t = PyJuliaBaseValue_Type() + isnull(t) && return PyPtr() + PyJuliaValue_New(t, v) +end + +PyJuliaValue_TryConvert_any(o, ::Type{S}) where {S} = begin + x = PyJuliaValue_GetValue(o) + putresult(tryconvert(S, x)) +end diff --git a/src/cpython/juliadict.jl b/src/cpython/juliadict.jl new file mode 100644 index 00000000..c8ba2296 --- /dev/null +++ b/src/cpython/juliadict.jl @@ -0,0 +1,85 @@ +const PyJuliaDictValue_Type__ref = Ref(PyPtr()) +PyJuliaDictValue_Type() = begin + ptr = PyJuliaDictValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.DictValue", + base = base, + iter = pyjldict_iter, + methods = [ + (name="keys", flags=Py_METH_NOARGS, meth=pyjldict_keys), + (name="values", flags=Py_METH_NOARGS, meth=pyjldict_values), + (name="items", flags=Py_METH_NOARGS, meth=pyjldict_items), + (name="get", flags=Py_METH_VARARGS, meth=pyjldict_get), + ], + as_sequence = ( + contains = pyjldict_contains, + ), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyMutableMappingABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaDictValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaDictValue_New(x::AbstractDict) = PyJuliaValue_New(PyJuliaDictValue_Type(), x) +PyJuliaValue_From(x::AbstractDict) = PyJuliaDictValue_New(x) + +pyjldict_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator(keys(PyJuliaValue_GetValue(xo)::AbstractDict))) + +pyjldict_keys(xo::PyPtr, ::PyPtr) = PyObject_From(keys(PyJuliaValue_GetValue(xo))) +pyjldict_values(xo::PyPtr, ::PyPtr) = PyObject_From(values(PyJuliaValue_GetValue(xo))) +pyjldict_items(xo::PyPtr, ::PyPtr) = PyObject_From(DictPairSet(pairs(PyJuliaValue_GetValue(xo)))) + +pyjldict_contains(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractDict + r = PyObject_TryConvert(vo, keytype(x)) + r == -1 && return Cint(-1) + r == 0 && return Cint(0) + v = takeresult(keytype(x)) + try + Cint(haskey(x, v)) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +pyjldict_get(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractDict + ism1(PyArg_CheckNumArgsBetween("get", args, 1, 2)) && return PyPtr() + ko = PyTuple_GetItem(args, 0) + vo = PyTuple_Size(args) < 2 ? Py_None() : PyTuple_GetItem(args, 1) + r = PyObject_TryConvert(ko, keytype(x)) + r == -1 && return PyPtr() + r == 0 && (Py_IncRef(vo); return vo) + k = takeresult(keytype(x)) + try + if haskey(x, k) + PyObject_From(x[k]) + else + Py_IncRef(vo) + vo + end + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +struct DictPairSet{K,V,T<:AbstractDict{K,V}} <: AbstractSet{Tuple{K,V}} + dict :: T +end +Base.length(x::DictPairSet) = length(x.dict) +Base.iterate(x::DictPairSet) = (r=iterate(x.dict); r===nothing ? nothing : (Tuple(r[1]), r[2])) +Base.iterate(x::DictPairSet, st) = (r=iterate(x.dict, st); r===nothing ? nothing : (Tuple(r[1]), r[2])) +Base.in(v, x::DictPairSet) = v in x.dict diff --git a/src/cpython/juliaerror.jl b/src/cpython/juliaerror.jl new file mode 100644 index 00000000..dd346c09 --- /dev/null +++ b/src/cpython/juliaerror.jl @@ -0,0 +1,69 @@ +const PyExc_JuliaError__ref = Ref(PyPtr()) +PyExc_JuliaError() = begin + ptr = PyExc_JuliaError__ref[] + if isnull(ptr) + c = [] + t = fill(PyType_Create(c, + name = "julia.Error", + base = PyExc_BaseException(), + str = pyjlerr_str, + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyExc_JuliaError__ref[] = ptr + end + ptr +end + +PyErr_SetJuliaError(err, bt=nothing) = begin + t = PyExc_JuliaError() + if isnull(t) + PyErr_Clear() + PyErr_SetString(PyExc_Exception(), "Julia error: $err (an error occurred while setting this error)") + else + if bt === nothing + bt = catch_backtrace() + end + v = PyJuliaValue_From((err, bt)) + if isnull(v) + PyErr_Clear() + PyErr_SetString(t, string(err)) + else + PyErr_SetObject(t, v) + Py_DecRef(v) + end + end +end + +PyErr_SetStringFromJuliaError(t, err) = begin + io = IOBuffer() + showerror(io, err) + msg = String(take!(io)) + PyErr_SetString(t, "Julia: $msg") +end + +pyjlerr_str(xo::PyPtr) = begin + args = PyObject_GetAttrString(xo, "args") + isnull(args) && return PyPtr() + r = PyObject_TryConvert(args, Tuple{Union{String, Tuple}}) + r == -1 && (Py_DecRef(args); return PyPtr()) + r == 0 && @goto fallback + (x,) = takeresult(Tuple{Union{String, Tuple}}) + if x isa String + Py_DecRef(args) + return PyUnicode_From(x) + elseif x isa Tuple{Exception, Any} + Py_DecRef(args) + io = IOBuffer() + showerror(io, x[1]) + msg = String(take!(io)) + return PyUnicode_From(msg) + end + + @label fallback + so = PyObject_Str(args) + Py_DecRef(args) + return so +end diff --git a/src/cpython/juliaiterator.jl b/src/cpython/juliaiterator.jl new file mode 100644 index 00000000..7a971921 --- /dev/null +++ b/src/cpython/juliaiterator.jl @@ -0,0 +1,57 @@ +mutable struct Iterator + val :: Any + st :: Any +end +Iterator(x) = Iterator(x, nothing) +Base.length(x::Iterator) = length(x.val) + +const PyJuliaIteratorValue_Type__ref = Ref(PyPtr()) +PyJuliaIteratorValue_Type() = begin + ptr = PyJuliaIteratorValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.IteratorValue", + base = base, + iter = pyjliter_iter, + iternext = pyjliter_iternext, + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaIteratorValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaIteratorValue_New(x::Iterator) = PyJuliaValue_New(PyJuliaIteratorValue_Type(), x) + +pyjliter_iter(xo::PyPtr) = PyJuliaIteratorValue_New(Iterator((PyJuliaValue_GetValue(xo)::Iterator).val)) + +pyjliter_iternext(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::Iterator + val = x.val + st = x.st + if st === nothing + z = iterate(val) + else + z = iterate(val, something(st)) + end + if z === nothing + PyPtr() + else + r, newst = z + x.st = Some(newst) + PyObject_From(r) + end +catch err + if err isa MethodError && err.f === iterate + PyErr_SetStringFromJuliaError(PyExc_TypeError(), err) + else + PyErr_SetJuliaError(err) + end + PyPtr() +end diff --git a/src/cpython/juliamodule.jl b/src/cpython/juliamodule.jl new file mode 100644 index 00000000..29ed622a --- /dev/null +++ b/src/cpython/juliamodule.jl @@ -0,0 +1,53 @@ +const PyJuliaModuleValue_Type__ref = Ref(PyPtr()) +PyJuliaModuleValue_Type() = begin + ptr = PyJuliaModuleValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.ModuleValue", + base = base, + methods = [ + (name="eval", flags=Py_METH_VARARGS, meth=pyjlmodule_eval), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaModuleValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaModuleValue_New(x::Module) = PyJuliaValue_New(PyJuliaModuleValue_Type(), x) +PyJuliaValue_From(x::Module) = PyJuliaModuleValue_New(x) + +pyjlmodule_eval(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::Module + ism1(PyArg_CheckNumArgsBetween("eval", args, 1, 2)) && return PyPtr() + if PyTuple_Size(args) == 1 + m = PyJuliaValue_GetValue(xo)::Module + exo = PyTuple_GetItem(args, 0) + else + ism1(PyArg_GetArg(Module, "eval", args, 0)) && return PyPtr() + m = takeresult(Module) + exo = PyTuple_GetItem(args, 1) + end + try + if PyUnicode_Check(exo) + c = PyUnicode_AsString(exo) + isempty(c) && PyErr_IsSet() && return PyPtr() + ex = Meta.parse(c) + else + ism1(PyObject_Convert(exo, Any)) && return PyPtr() + ex = takeresult() + end + r = Base.eval(m, ex) + PyObject_From(r) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end diff --git a/src/cpython/juliaraw.jl b/src/cpython/juliaraw.jl new file mode 100644 index 00000000..b26158fd --- /dev/null +++ b/src/cpython/juliaraw.jl @@ -0,0 +1,206 @@ +const PyJuliaRawValue_Type__ref = Ref(PyPtr()) +PyJuliaRawValue_Type() = begin + ptr = PyJuliaRawValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaBaseValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.RawValue", + base = base, + repr = pyjlraw_repr, + str = pyjlraw_str, + getattro = pyjlraw_getattro, + setattro = pyjlraw_setattro, + call = pyjlraw_call, + as_mapping = ( + length = pyjlraw_length, + subscript = pyjlraw_getitem, + ass_subscript = pyjlraw_setitem, + ), + methods = [ + (name="__dir__", flags=Py_METH_NOARGS, meth=pyjlraw_dir), + ] + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaRawValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaRawValue_New(x) = PyJuliaValue_New(PyJuliaRawValue_Type(), x) + +pyjlraw_repr(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = "" + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + return PyPtr() +end + +pyjlraw_str(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + s = string(x) + PyUnicode_From(s) +catch err + PyErr_SetJuliaError(err) + return PyPtr() +end + +pyjl_attr_py2jl(k::String) = replace(k, r"_[b]+$" => (x -> "!"^(length(x)-1))) +pyjl_attr_jl2py(k::String) = replace(k, r"!+$" => (x -> "_" * "b"^length(x))) + +pyjlraw_getattro(xo::PyPtr, ko::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericGetAttr(xo, ko) + if isnull(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to get the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return PyPtr() + k = pyjl_attr_py2jl(k) + try + v = getproperty(x, Symbol(k)) + PyJuliaRawValue_New(v) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlraw_setattro(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + # Try generic lookup first + ro = PyObject_GenericSetAttr(xo, ko, vo) + if ism1(ro) && PyErr_IsSet(PyExc_AttributeError()) + PyErr_Clear() + else + return ro + end + # Now try to set the corresponding property + x = PyJuliaValue_GetValue(xo) + k = PyUnicode_AsString(ko) + isempty(k) && PyErr_IsSet() && return Cint(-1) + k = pyjl_attr_py2jl(k) + ism1(PyObject_Convert(vo, Any)) && return Cint(-1) + v = takeresult(Any) + try + setproperty!(x, Symbol(k), v) + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end +end + +pyjlraw_dir(xo::PyPtr, _::PyPtr) = begin + fo = PyObject_GetAttrString(PyJuliaBaseValue_Type(), "__dir__") + isnull(fo) && return PyPtr() + ro = PyObject_CallNice(fo, PyObjectRef(xo)) + Py_DecRef(fo) + isnull(ro) && return PyPtr() + x = PyJuliaValue_GetValue(xo) + ks = try + collect(map(string, propertynames(x))) + catch err + Py_DecRef(ro) + PyErr_SetJuliaError(err) + return PyPtr() + end + for k in ks + ko = PyUnicode_From(pyjl_attr_jl2py(k)) + isnull(ko) && (Py_DecRef(ro); return PyPtr()) + err = PyList_Append(ro, ko) + Py_DecRef(ko) + ism1(err) && (Py_DecRef(ro); return PyPtr()) + end + return ro +end + +pyjlraw_call(fo::PyPtr, argso::PyPtr, kwargso::PyPtr) = begin + f = PyJuliaValue_GetValue(fo) + if isnull(argso) + args = Vector{Any}() + else + ism1(PyObject_Convert(argso, Vector{Any})) && return PyPtr() + args = takeresult(Vector{Any}) + end + if isnull(kwargso) + kwargs = Dict{Symbol, Any}() + else + ism1(PyObject_Convert(kwargso, Dict{Symbol, Any})) && return PyPtr() + kwargs = takeresult(Dict{Symbol, Any}) + end + try + x = f(args...; kwargs...) + PyJuliaRawValue_New(x) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlraw_length(xo::PyPtr) = try + x = PyJuliaValue_GetValue(xo) + Py_ssize_t(length(x)) +catch err + PyErr_SetJuliaError(err) + Py_ssize_t(-1) +end + +pyjlraw_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + if PyTuple_Check(ko) + ism1(PyObject_Convert(ko, Tuple)) && return PyPtr() + k = takeresult(Tuple) + try + PyJuliaRawValue_New(x[k...]) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + else + ism1(PyObject_Convert(ko, Any)) && return PyPtr() + k = takeresult(Any) + try + PyJuliaRawValue_New(x[k]) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + end +end + +pyjlraw_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo) + ism1(PyObject_Convert(vo, Any)) && return PyPtr() + v = takeresult(Any) + if PyTuple_Check(ko) + ism1(PyObject_Convert(ko, Tuple)) && return PyPtr() + k = takeresult(Tuple) + try + x[k...] = v + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end + else + ism1(PyObject_Convert(ko, Any)) && return PyPtr() + k = takeresult(Any) + try + x[k] = v + Cint(0) + catch err + PyErr_SetJuliaError(err) + Cint(-1) + end + end +end diff --git a/src/cpython/juliatype.jl b/src/cpython/juliatype.jl new file mode 100644 index 00000000..8ca8ffde --- /dev/null +++ b/src/cpython/juliatype.jl @@ -0,0 +1,56 @@ +const PyJuliaTypeValue_Type__ref = Ref(PyPtr()) +PyJuliaTypeValue_Type() = begin + ptr = PyJuliaTypeValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaAnyValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.TypeValue", + base = base, + as_mapping = ( + subscript = pyjltype_getitem, + ass_subscript = pyjltype_setitem, + ), + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaTypeValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaTypeValue_New(x::Type) = PyJuliaValue_New(PyJuliaTypeValue_Type(), x) +PyJuliaValue_From(x::Type) = PyJuliaTypeValue_New(x) + +pyjltype_getitem(xo::PyPtr, ko::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::Type + if PyTuple_Check(ko) + r = PyObject_Convert(ko, Tuple) + ism1(r) && return PyPtr() + k = takeresult(Tuple) + try + PyObject_From(x{k...}) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + else + r = PyObject_Convert(ko, Any) + ism1(r) && return PyPtr() + k = takeresult(Any) + try + PyObject_From(x{k}) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end + end +end + +pyjltype_setitem(xo::PyPtr, ko::PyPtr, vo::PyPtr) = begin + PyErr_SetString(PyExc_TypeError(), "Not supported.") + PyErr() +end diff --git a/src/cpython/juliavector.jl b/src/cpython/juliavector.jl new file mode 100644 index 00000000..fb246290 --- /dev/null +++ b/src/cpython/juliavector.jl @@ -0,0 +1,233 @@ +const PyJuliaVectorValue_Type__ref = Ref(PyPtr()) +PyJuliaVectorValue_Type() = begin + ptr = PyJuliaVectorValue_Type__ref[] + if isnull(ptr) + c = [] + base = PyJuliaArrayValue_Type() + isnull(base) && return PyPtr() + t = fill(PyType_Create(c, + name = "julia.VectorValue", + base = base, + methods = [ + (name="resize", flags=Py_METH_O, meth=pyjlvector_resize), + (name="sort", flags=Py_METH_KEYWORDS|Py_METH_VARARGS, meth=pyjlvector_sort), + (name="reverse", flags=Py_METH_NOARGS, meth=pyjlvector_reverse), + (name="clear", flags=Py_METH_NOARGS, meth=pyjlvector_clear), + (name="__reversed__", flags=Py_METH_NOARGS, meth=pyjlvector_reversed), + (name="insert", flags=Py_METH_VARARGS, meth=pyjlvector_insert), + (name="append", flags=Py_METH_O, meth=pyjlvector_append), + (name="extend", flags=Py_METH_O, meth=pyjlvector_extend), + (name="pop", flags=Py_METH_VARARGS, meth=pyjlvector_pop), + (name="remove", flags=Py_METH_O, meth=pyjlvector_remove), + (name="index", flags=Py_METH_O, meth=pyjlvector_index), + (name="count", flags=Py_METH_O, meth=pyjlvector_count), + ], + )) + ptr = PyPtr(pointer(t)) + err = PyType_Ready(ptr) + ism1(err) && return PyPtr() + abc = PyMutableSequenceABC_Type() + isnull(abc) && return PyPtr() + ism1(PyABC_Register(ptr, abc)) && return PyPtr() + PYJLGCCACHE[ptr] = push!(c, t) + PyJuliaVectorValue_Type__ref[] = ptr + end + ptr +end + +PyJuliaVectorValue_New(x::AbstractVector) = PyJuliaValue_New(PyJuliaVectorValue_Type(), x) +PyJuliaValue_From(x::AbstractVector) = PyJuliaVectorValue_New(x) + +pyjlvector_resize(xo::PyPtr, arg::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(arg, Int) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "size must be an integer"); return PyPtr()) + resize!(x, takeresult(Int)) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_sort(xo::PyPtr, args::PyPtr, kwargs::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + ism1(PyArg_CheckNumArgsEq("sort", args, 0)) && return PyPtr() + ism1(PyArg_GetArg(Bool, "sort", kwargs, "reverse", false)) && return PyPtr() + rev = takeresult(Bool) + ism1(PyArg_GetArg(Any, "sort", kwargs, "key", nothing)) && return PyPtr() + by = takeresult() + sort!(x, rev=rev, by= by===nothing ? identity : by) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_reverse(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + reverse!(x) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_clear(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + empty!(x) + PyNone_New() +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_reversed(xo::PyPtr, ::PyPtr) = try + x = PyJuliaValue_GetValue(xo)::AbstractVector + PyObject_From(reverse(x)) +catch err + PyErr_SetJuliaError(err) + PyPtr() +end + +pyjlvector_insert(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + a = axes(x, 1) + ism1(PyArg_CheckNumArgsEq("insert", args, 2)) && return PyPtr() + ism1(PyArg_GetArg(Int, "insert", args, 0)) && return PyPtr() + k = takeresult(Int) + k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PyPtr()) + ism1(PyArg_GetArg(eltype(x), "insert", args, 1)) && return PyPtr() + v = takeresult(eltype(x)) + try + insert!(x, k′, v) + PyNone_New() + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_append(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "array value must be a Julia '$(eltype(x))', not a '$(PyType_Name(Py_Type(vo)))'"); return PyPtr()) + v = takeresult(eltype(x)) + try + push!(x, v) + PyNone_New() + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_extend(xo::PyPtr, vso::PyPtr) = begin + # TODO: this might be quicker if we introduce a function barrier so typeof(x) is known + x = PyJuliaValue_GetValue(xo)::AbstractVector + it = PyObject_GetIter(vso) + isnull(it) && return PyPtr() + try + while true + vo = PyIter_Next(it) + if !isnull(vo) + r = PyObject_TryConvert(vo, eltype(x)) + Py_DecRef(vo) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_TypeError(), "array value must be a Julia '$(eltype(x))', not a '$(PyType_Name(Py_Type(vo)))'"); return PyPtr()) + push!(x, takeresult(eltype(x))) + elseif PyErr_IsSet() + return PyPtr() + else + return PyNone_New() + end + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + finally + Py_DecRef(it) + end +end + +pyjlvector_pop(xo::PyPtr, args::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + a = axes(x, 1) + ism1(PyArg_CheckNumArgsLe("pop", args, 1)) && return PyPtr() + ism1(PyArg_GetArg(Int, "pop", args, 0, -1)) && return PyPtr() + k = takeresult(Int) + k′ = k < 0 ? (last(a) + 1 + k) : (first(a) + k) + checkbounds(Bool, x, k′) || (PyErr_SetString(PyExc_IndexError(), "array index out of bounds"); return PyPtr()) + try + if k′ == last(a) + v = pop!(x) + elseif k′ == first(a) + v = popfirst!(x) + else + v = x[k′] + deleteat!(x, k′) + end + PyObject_From(v) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_remove(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_ValueError(), "value not in vector"); return PyPtr()) + v = takeresult(eltype(x)) + try + k = findfirst(x->x==v, x) + if k === nothing + PyErr_SetString(PyExc_ValueError(), "value not in vector") + PyPtr() + else + deleteat!(x, k) + PyNone_New() + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_index(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && (PyErr_SetString(PyExc_ValueError(), "value not in vector"); return PyPtr()) + v = takeresult(eltype(x)) + try + k = findfirst(x->x==v, x) + if k === nothing + PyErr_SetString(PyExc_ValueError(), "value not in vector") + PyPtr() + else + PyObject_From(k - first(axes(x, 1))) + end + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end + +pyjlvector_count(xo::PyPtr, vo::PyPtr) = begin + x = PyJuliaValue_GetValue(xo)::AbstractVector + r = PyObject_TryConvert(vo, eltype(x)) + r == -1 && return PyPtr() + r == 0 && return PyObject_From(0) + v = takeresult(eltype(x)) + try + n = count(x->x==v, x) + PyObject_From(n) + catch err + PyErr_SetJuliaError(err) + PyPtr() + end +end diff --git a/src/cpython/list.jl b/src/cpython/list.jl new file mode 100644 index 00000000..7e1fff71 --- /dev/null +++ b/src/cpython/list.jl @@ -0,0 +1,31 @@ +@cdef :PyList_New PyPtr (Py_ssize_t,) +@cdef :PyList_Append Cint (PyPtr, PyPtr) +@cdef :PyList_AsTuple PyPtr (PyPtr,) + +const PyList_Type__ref = Ref(PyPtr()) +PyList_Type() = pyglobal(PyList_Type__ref, :PyList_Type) + +PyList_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_LIST_SUBCLASS) + +PyList_CheckExact(o) = Py_TypeCheckExact(o, PyList_Type()) + +PyList_From(xs::Union{Tuple,AbstractVector}) = PyList_FromIter(xs) + +PyList_FromIter(xs) = begin + r = PyList_New(0) + isnull(r) && return PyPtr() + try + for (i,x) in enumerate(xs) + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(r); return PyPtr()) + err = PyList_Append(r, xo) + Py_DecRef(xo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch err + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end diff --git a/src/cpython/mapping.jl b/src/cpython/mapping.jl new file mode 100644 index 00000000..f7e0b985 --- /dev/null +++ b/src/cpython/mapping.jl @@ -0,0 +1,3 @@ +@cdef :PyMapping_HasKeyString Cint (PyPtr, Cstring) +@cdef :PyMapping_SetItemString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyMapping_GetItemString PyPtr (PyPtr, Cstring) diff --git a/src/cpython/method.jl b/src/cpython/method.jl new file mode 100644 index 00000000..4b1605f6 --- /dev/null +++ b/src/cpython/method.jl @@ -0,0 +1 @@ +@cdef :PyInstanceMethod_New PyPtr (PyPtr,) diff --git a/src/cpython/newtype.jl b/src/cpython/newtype.jl new file mode 100644 index 00000000..74571038 --- /dev/null +++ b/src/cpython/newtype.jl @@ -0,0 +1,139 @@ +### CACHE + +cacheptr!(c, x::Ptr) = x +cacheptr!(c, x::String) = (push!(c, x); pointer(x)) +cacheptr!(c, x::AbstractString) = cacheptr!(c, String(x)) +cacheptr!(c, x::Array) = (push!(c, x); pointer(x)) +cacheptr!(c, x::AbstractArray) = cacheptr!(c, Array(x)) +cacheptr!(c, x::PyObject) = (push!(c, x); pyptr(x)) +cacheptr!(c, x::Base.CFunction) = (push!(c, x); Base.unsafe_convert(Ptr{Cvoid}, x)) + +cachestrptr!(c, x::Ptr) = x +cachestrptr!(c, x) = cacheptr!(c, string(x)) + +macro cachefuncptr!(c, f, R, Ts) + quote + let c=$(esc(c)), f=$(esc(f)) + if f isa Ptr + f + elseif f isa Base.CFunction + cacheptr!(c, f) + else + cacheptr!(c, @cfunction($(Expr(:$, :f)), $(esc(R)), ($(map(esc, Ts.args)...),))) + end + end + end +end + +### PROTOCOLS + +PyNumberMethods_Create(c, x::PyNumberMethods) = x +PyNumberMethods_Create(c; opts...) = C.PyNumberMethods(; [k => (v isa Ptr ? v : v isa Base.CFunction ? cacheptr!(c, v) : error()) for (k,v) in pairs(opts)]...) +PyNumberMethods_Create(c, x::Dict) = PyNumberMethods_Create(c; x...) +PyNumberMethods_Create(c, x::NamedTuple) = PyNumberMethods_Create(c; x...) + +PyMappingMethods_Create(c, x::PyMappingMethods) = x +PyMappingMethods_Create(c; length=C_NULL, subscript=C_NULL, ass_subscript=C_NULL) = + PyMappingMethods( + length = @cachefuncptr!(c, length, Py_ssize_t, (PyPtr,)), + subscript = @cachefuncptr!(c, subscript, PyPtr, (PyPtr, PyPtr)), + ass_subscript = @cachefuncptr!(c, ass_subscript, Cint, (PyPtr, PyPtr, PyPtr)), + ) +PyMappingMethods_Create(c, x::Dict) = PyMappingMethods_Create(c; x...) +PyMappingMethods_Create(c, x::NamedTuple) = PyMappingMethods_Create(c; x...) + +PySequenceMethods_Create(c, x::PySequenceMethods) = x +PySequenceMethods_Create(c; + length=C_NULL, concat=C_NULL, repeat=C_NULL, item=C_NULL, ass_item=C_NULL, + contains=C_NULL, inplace_concat=C_NULL, inplace_repeat=C_NULL, +) = + PySequenceMethods( + length = @cachefuncptr!(c, length, Py_ssize_t, (PyPtr,)), + concat = @cachefuncptr!(c, concat, PyPtr, (PyPtr, PyPtr)), + repeat = @cachefuncptr!(c, repeat, PyPtr, (PyPtr, Py_ssize_t)), + item = @cachefuncptr!(c, item, PyPtr, (PyPtr, Py_ssize_t)), + ass_item = @cachefuncptr!(c, ass_item, Cint, (PyPtr, Py_ssize_t, PyPtr)), + contains = @cachefuncptr!(c, contains, Cint, (PyPtr, PyPtr)), + inplace_concat = @cachefuncptr!(c, inplace_concat, PyPtr, (PyPtr, PyPtr)), + inplace_repeat = @cachefuncptr!(c, inplace_repeat, PyPtr, (PyPtr, Py_ssize_t)), + ) +PySequenceMethods_Create(c, x::Dict) = PySequenceMethods_Create(c; x...) +PySequenceMethods_Create(c, x::NamedTuple) = PySequenceMethods_Create(c; x...) + +PyBufferProcs_Create(c, x::PyBufferProcs) = x +PyBufferProcs_Create(c; get=C_NULL, release=C_NULL) = + PyBufferProcs( + get = @cachefuncptr!(c, get, Cint, (PyPtr, Ptr{Py_buffer}, Cint)), + release = @cachefuncptr!(c, release, Cvoid, (PyPtr, Ptr{Py_buffer})), + ) +PyBufferProcs_Create(c, x::Dict) = PyBufferProcs_Create(c; x...) +PyBufferProcs_Create(c, x::NamedTuple) = PyBufferProcs_Create(c; x...) + +PyMethodDef_Create(c, x::PyMethodDef) = x +PyMethodDef_Create(c; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = + PyMethodDef( + name = cachestrptr!(c, name), + meth = iszero(flags & Py_METH_KEYWORDS) ? @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr)) : @cachefuncptr!(c, meth, PyPtr, (PyPtr, PyPtr, PyPtr)), + flags = flags, + doc = cachestrptr!(c, doc), + ) +PyMethodDef_Create(c, x::Dict) = PyMethodDef_Create(c; x...) +PyMethodDef_Create(c, x::NamedTuple) = PyMethodDef_Create(c; x...) + +PyGetSetDef_Create(c, x::PyGetSetDef) = x +PyGetSetDef_Create(c; name=C_NULL, get=C_NULL, set=C_NULL, doc=C_NULL, closure=C_NULL) = + PyGetSetDef( + name = cachestrptr!(c, name), + get = @cachefuncptr!(c, get, PyPtr, (PyPtr, Ptr{Cvoid})), + set = @cachefuncptr!(c, set, Cint, (PyPtr, PyPtr, Ptr{Cvoid})), + doc = cachestrptr!(c, doc), + closure = closure, + ) +PyGetSetDef_Create(c, x::Dict) = PyGetSetDef_Create(c; x...) +PyGetSetDef_Create(c, x::NamedTuple) = PyGetSetDef_Create(c; x...) + +PyType_Create(c; + type=C_NULL, name, as_number=C_NULL, as_mapping=C_NULL, as_sequence=C_NULL, + as_buffer=C_NULL, methods=C_NULL, getset=C_NULL, dealloc=C_NULL, getattr=C_NULL, + setattr=C_NULL, repr=C_NULL, hash=C_NULL, call=C_NULL, str=C_NULL, getattro=C_NULL, + setattro=C_NULL, doc=C_NULL, iter=C_NULL, iternext=C_NULL, richcompare=C_NULL, opts... +) = begin + type = cacheptr!(c, type) + name = cachestrptr!(c, name) + as_number = as_number isa Ptr ? as_number : cacheptr!(c, fill(PyNumberMethods_Create(c, as_number))) + as_mapping = as_mapping isa Ptr ? as_mapping : cacheptr!(c, fill(PyMappingMethods_Create(c, as_mapping))) + as_sequence = as_sequence isa Ptr ? as_sequence : cacheptr!(c, fill(PySequenceMethods_Create(c, as_sequence))) + as_buffer = as_buffer isa Ptr ? as_buffer : cacheptr!(c, fill(PyBufferProcs_Create(c, as_buffer))) + methods = + if methods isa Ptr + methods + else + cacheptr!(c, [[PyMethodDef_Create(c, m) for m in methods]; PyMethodDef()]) + end + getset = + if getset isa Ptr + getset + else + cacheptr!(c, [[PyGetSetDef_Create(c, m) for m in getset]; PyGetSetDef()]) + end + dealloc = @cachefuncptr!(c, dealloc, Cvoid, (PyPtr,)) + getattr = @cachefuncptr!(c, getattr, PyPtr, (PyPtr, Cstring)) + setattr = @cachefuncptr!(c, setattr, Cint, (PyPtr, Cstring, PyPtr)) + repr = @cachefuncptr!(c, repr, PyPtr, (PyPtr,)) + hash = @cachefuncptr!(c, hash, Py_hash_t, (PyPtr,)) + call = @cachefuncptr!(c, call, PyPtr, (PyPtr, PyPtr, PyPtr)) + str = @cachefuncptr!(c, str, PyPtr, (PyPtr,)) + getattro = @cachefuncptr!(c, getattro, PyPtr, (PyPtr, PyPtr)) + setattro = @cachefuncptr!(c, setattro, Cint, (PyPtr, PyPtr, PyPtr)) + doc = cachestrptr!(c, doc) + iter = @cachefuncptr!(c, iter, PyPtr, (PyPtr,)) + iternext = @cachefuncptr!(c, iternext, PyPtr, (PyPtr,)) + richcompare = @cachefuncptr!(c, richcompare, PyPtr, (PyPtr, PyPtr, Cint)) + PyTypeObject(; + ob_base=PyVarObject(ob_base=PyObject(type=type)), name=name, as_number=as_number, + as_mapping=as_mapping, as_sequence=as_sequence, as_buffer=as_buffer, + methods=methods, getset=getset, dealloc=dealloc, getattr=getattr, setattr=setattr, + repr=repr, hash=hash, call=call, str=str, getattro=getattro, setattro=setattro, + iter=iter, iternext=iternext, richcompare=richcompare, opts... + ) +end diff --git a/src/cpython/none.jl b/src/cpython/none.jl new file mode 100644 index 00000000..f3da54df --- /dev/null +++ b/src/cpython/none.jl @@ -0,0 +1,16 @@ +const Py_NotImplemented__ref = Ref(PyPtr()) +Py_NotImplemented() = pyglobal(Py_NotImplemented__ref, :_Py_NotImplementedStruct) + +PyNotImplemented_Check(o) = Py_Is(o, Py_NotImplemented()) + +PyNotImplemented_New() = (o=Py_NotImplemented(); Py_IncRef(o); o) + +const Py_None__ref = Ref(PyPtr()) +Py_None() = pyglobal(Py_None__ref, :_Py_NoneStruct) + +PyNone_Check(o) = Py_Is(o, Py_None()) + +PyNone_New() = (o=Py_None(); Py_IncRef(o); o) + +PyNone_TryConvertRule_nothing(o, ::Type{Nothing}) = putresult(nothing) +PyNone_TryConvertRule_missing(o, ::Type{Missing}) = putresult(missing) diff --git a/src/cpython/number.jl b/src/cpython/number.jl new file mode 100644 index 00000000..7fa2ac1e --- /dev/null +++ b/src/cpython/number.jl @@ -0,0 +1,57 @@ +@cdef :PyNumber_Add PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Subtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Multiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_MatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_FloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_TrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Remainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_DivMod PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Power PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_Negative PyPtr (PyPtr,) +@cdef :PyNumber_Positive PyPtr (PyPtr,) +@cdef :PyNumber_Absolute PyPtr (PyPtr,) +@cdef :PyNumber_Invert PyPtr (PyPtr,) +@cdef :PyNumber_Lshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Rshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_And PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Xor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Or PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAdd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceSubtract PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceMatrixMultiply PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceFloorDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceTrueDivide PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRemainder PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlacePower PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyNumber_InPlaceLshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceRshift PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceAnd PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceXor PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_InPlaceOr PyPtr (PyPtr, PyPtr) +@cdef :PyNumber_Long PyPtr (PyPtr,) +@cdef :PyNumber_Float PyPtr (PyPtr,) +@cdef :PyNumber_Index PyPtr (PyPtr,) + +for n in [:Number, :Complex, :Real, :Rational, :Integral] + p = Symbol(:Py, n, :ABC) + t = Symbol(p, :_Type) + tr = Symbol(p, :__ref) + c = Symbol(p, :_Check) + @eval const $tr = Ref(PyPtr()) + @eval $t(doimport::Bool=true) = begin + ptr = $tr[] + isnull(ptr) || return ptr + a = doimport ? PyImport_ImportModule("numbers") : PyImport_GetModule("numbers") + isnull(a) && return a + b = PyObject_GetAttrString(a, $(string(n))) + Py_DecRef(a) + isnull(b) && return b + $tr[] = b + end + @eval $c(o) = begin + t = $t(false) + isnull(t) && return (PyErr_IsSet() ? Cint(-1) : Cint(0)) + PyObject_IsInstance(o, t) + end +end diff --git a/src/cpython/numpy.jl b/src/cpython/numpy.jl new file mode 100644 index 00000000..c2637951 --- /dev/null +++ b/src/cpython/numpy.jl @@ -0,0 +1,6 @@ +struct PyNumpySimpleData_TryConvert_value{R,tr} end + +(::PyNumpySimpleData_TryConvert_value{R,tr})(o, ::Type{S}) where {S,R,tr} = begin + val = PySimpleObject_GetValue(o, R) + putresult(tr ? tryconvert(S, val) : convert(S, val)) +end diff --git a/src/cpython/object.jl b/src/cpython/object.jl new file mode 100644 index 00000000..466113d5 --- /dev/null +++ b/src/cpython/object.jl @@ -0,0 +1,297 @@ +@cdef :_PyObject_New PyPtr (PyPtr,) +@cdef :PyObject_ClearWeakRefs Cvoid (PyPtr,) +@cdef :PyObject_HasAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_HasAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_GetAttrString PyPtr (PyPtr, Cstring) +@cdef :PyObject_GetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_GenericGetAttr PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetAttrString Cint (PyPtr, Cstring, PyPtr) +@cdef :PyObject_SetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_GenericSetAttr Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelAttrString Cint (PyPtr, Cstring) +@cdef :PyObject_DelAttr Cint (PyPtr, PyPtr) +@cdef :PyObject_RichCompare PyPtr (PyPtr, PyPtr, Cint) +@cdef :PyObject_RichCompareBool Cint (PyPtr, PyPtr, Cint) +@cdef :PyObject_Repr PyPtr (PyPtr,) +@cdef :PyObject_ASCII PyPtr (PyPtr,) +@cdef :PyObject_Str PyPtr (PyPtr,) +@cdef :PyObject_Bytes PyPtr (PyPtr,) +@cdef :PyObject_IsSubclass Cint (PyPtr, PyPtr) +@cdef :PyObject_IsInstance Cint (PyPtr, PyPtr) +@cdef :PyObject_Hash Py_hash_t (PyPtr,) +@cdef :PyObject_IsTrue Cint (PyPtr,) +@cdef :PyObject_Length Py_ssize_t (PyPtr,) +@cdef :PyObject_GetItem PyPtr (PyPtr, PyPtr) +@cdef :PyObject_SetItem Cint (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_DelItem Cint (PyPtr, PyPtr) +@cdef :PyObject_Dir PyPtr (PyPtr,) +@cdef :PyObject_GetIter PyPtr (PyPtr,) +@cdef :PyObject_Call PyPtr (PyPtr, PyPtr, PyPtr) +@cdef :PyObject_CallObject PyPtr (PyPtr, PyPtr) + +const PyObject_Type__ref = Ref(PyPtr()) +PyObject_Type() = pyglobal(PyObject_Type__ref, :PyBaseObject_Type) + +PyObject_From(x::PyObjectRef) = (Py_IncRef(x.ptr); x.ptr) +PyObject_From(x::Nothing) = PyNone_New() +PyObject_From(x::Missing) = PyNone_New() +PyObject_From(x::Bool) = PyBool_From(x) +PyObject_From(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = PyLong_From(x) +PyObject_From(x::Rational{<:Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyFraction_From(x) +PyObject_From(x::Union{Float16,Float32,Float64}) = PyFloat_From(x) +PyObject_From(x::Complex{<:Union{Float16,Float32,Float64}}) = PyComplex_From(x) +PyObject_From(x::Union{String,SubString{String}}) = PyUnicode_From(x) +PyObject_From(x::Char) = PyUnicode_From(string(x)) +PyObject_From(x::Tuple) = PyTuple_From(x) +PyObject_From(x::AbstractRange{<:Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}}) = PyRange_From(x) +PyObject_From(x) = + if ispyreftype(typeof(x)) + GC.@preserve x begin + ptr = pyptr(x) + Py_IncRef(ptr) + ptr + end + else + PyJuliaValue_From(x) + # PyErr_SetString(PyExc_TypeError(), "Cannot convert this Julia '$(typeof(x))' to a Python object.") + # PyPtr() + end + +function PyObject_CallArgs(f, args::Tuple, kwargs::Union{Nothing,NamedTuple,Base.Iterators.Pairs{Symbol},Base.Iterators.Pairs{Union{}}}=nothing) + if kwargs!==nothing && !isempty(kwargs) + argso = PyTuple_From(args) + isnull(argso) && return PyPtr() + kwargso = PyDict_From(kwargs) + isnull(kwargso) && (Py_DecRef(argso); return PyPtr()) + r = PyObject_Call(f, argso, kwargso) + Py_DecRef(argso) + Py_DecRef(kwargso) + r + elseif !isempty(args) + argso = PyTuple_From(Tuple(args)) + isnull(argso) && return PyPtr() + r = PyObject_CallObject(f, argso) + Py_DecRef(argso) + return r + else + PyObject_CallObject(f, C_NULL) + end +end + +PyObject_CallNice(f, args...; kwargs...) = PyObject_CallArgs(f, args, kwargs) + +""" +Mapping of Julia types to mappings of Python types to vectors of compiled functions implementing the conversion. +""" +const TRYCONVERT_COMPILED_RULES = IdDict{Type, Dict{PyPtr, Vector{Ptr{Cvoid}}}}() + +const TRYCONVERT_COMPILED_RULES_CACHE = Dict{Any, PyPtr}() + +""" +Mapping of type names to lists of rules. + +A rule is a triple `(priority, type, impl)`. + +Rules are applied in priority order, then in MRO order, then in the order in this list. You should currently only use the following values: +- `0` (the default) for most rules. +- `100` for "canonical" rules which are the preferred way to convert a type, e.g. `float` to `Float64`. +- `-100` for "last ditch" rules once all other options have been exhausted (currently used for anything to `PyObject` or `PyRef`). + +The rule is applied only when the `type` has non-trivial intersection with the target type `T`. + +Finally `impl` is the function implementing the conversion. +Its signature is `(o, T, S)` where `o` is the `PyPtr` object to convert, `T` is the desired type, and `S = typeintersect(T, type)`. +On success it returns `putresult(x)` where `x::T` is the converted value (this stores the result and returns `1`). +If conversion was not possible, returns `0` (indicating we move on to the next rule in the list). +On error, returns `-1`. +""" +const TRYCONVERT_RULES = Dict{String, Vector{Tuple{Int, Type, Any}}}() + +""" +List of niladic functions returning a pointer to a type. We always check for subtypes of these types in TryConvert. + +Can also return NULL. Without an error set, this indicates the type is not loaded (e.g. its containing module is not loaded) and therefore is skipped over. +""" +const TRYCONVERT_EXTRATYPES = Vector{Any}() + +@generated PyObject_TryConvert_CompiledRules(::Type{T}) where {T} = + get!(Dict{PyPtr, Vector{Ptr{Cvoid}}}, TRYCONVERT_COMPILED_RULES, T) + +PyObject_TryConvert_Rules(n::String) = + get!(Vector{Tuple{Int, Type, Function}}, TRYCONVERT_RULES, n) +PyObject_TryConvert_AddRule(n::String, T, rule, priority=0) = + push!(PyObject_TryConvert_Rules(n), (priority, T, rule)) +PyObject_TryConvert_AddRules(n::String, xs) = + for x in xs + PyObject_TryConvert_AddRule(n, x...) + end +PyObject_TryConvert_AddExtraType(tfunc) = + push!(TRYCONVERT_EXTRATYPES, tfunc) +PyObject_TryConvert_AddExtraTypes(xs) = + for x in xs + PyObject_TryConvert_AddExtraType(x) + end + +PyObject_TryConvert_CompileRule(::Type{T}, t::PyPtr) where {T} = begin + + ### STAGE 1: Get a list of supertypes to consider. + # + # This includes the MRO of t, but also the MRO of any extra type (such as an ABC) + # that we explicitly check for because it may not appear the the MRO of t. + + # MRO of t + tmro = PyType_MROAsVector(t) + # find the "MROs" of all base types we are considering + basetypes = PyPtr[t] + basemros = Vector{PyPtr}[tmro] + for xtf in TRYCONVERT_EXTRATYPES + xt = xtf() + isnull(xt) && (PyErr_IsSet() ? (return PYERR()) : continue) + xb = PyPtr() + for b in tmro + r = PyObject_IsSubclass(b, xt) + ism1(r) && return PYERR() + r != 0 && (xb = b) + end + if xb != PyPtr() + push!(basetypes, xt) + push!(basemros, [xb; PyType_MROAsVector(xt)]) + end + end + for b in basetypes[2:end] + push!(basemros, [b]) + end + # merge the MROs + # this is a port of the merge() function at the bottom of: + # https://www.python.org/download/releases/2.3/mro/ + alltypes = PyPtr[] + while !isempty(basemros) + # find the first head not contained in any tail + ok = false + b = PyPtr() + for bmro in basemros + b = bmro[1] + if all(bmro -> b ∉ bmro[2:end], basemros) + ok = true + break + end + end + ok || error("Fatal inheritence error: could not merge MROs") + # add it to the list + push!(alltypes, b) + # remove it from consideration + for bmro in basemros + filter!(x -> x != b, bmro) + end + # remove empty lists + filter!(x -> !isempty(x), basemros) + end + # check the original MRO is preserved + @assert filter(x -> x in tmro, alltypes) == tmro + # some special cases + extranames = Dict() + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array_struct__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array_interface__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyObject_HasAttrString(b, "__array__") != 0 + push!(get!(Vector, extranames, i), "") + break + end + end + for (i,b) in reverse(collect(enumerate(alltypes))) + if PyType_CheckBuffer(b) + push!(get!(Vector, extranames, i), "") + break + end + end + allnames = [n for (i,t) in enumerate(alltypes) for n in [PyType_FullName(t); get(Vector, extranames, i)]] + + ### STAGE 2: Get a list of applicable conversion rules. + # + # These are the conversion rules of the types found above + + # gather rules of the form (priority, order, S, rule) from these types + rules = Tuple{Int, Int, Type, Any}[] + for n in allnames + for (p, S, r) in PyObject_TryConvert_Rules(n) + push!(rules, (p, length(rules)+1, S, r)) + end + end + # sort by priority + sort!(rules, by=x->(-x[1], x[2])) + # intersect S with T + rules = [typeintersect(S,T) => r for (p,i,S,r) in rules] + # discard rules where S is a subtype of the union of all previous S for rules with the same implementation + # in particular this removes rules with S==Union{} and removes duplicates + rules = [S=>rule for (i,(S,rule)) in enumerate(rules) if !(S <: Union{[S′ for (S′,rule′) in rules[1:i-1] if rule==rule′]...})] + + @debug "PYTHON CONVERSION FOR '$(PyType_FullName(t))' to '$T'" basetypes=map(PyType_FullName, basetypes) alltypes=allnames rules=rules + + ### STAGE 3: Define and compile functions implementing these rules. + + map(rules) do x + get!(TRYCONVERT_COMPILED_RULES_CACHE, x) do + S, rule = x + # make the function implementing the rule + rulefunc = @eval (o::PyPtr) -> $rule(o, $S) + # compile it + crulefunc = @cfunction($rulefunc, Int, (PyPtr,)) + # cache it + push!(CACHE, crulefunc) + # get the function pointer + Base.unsafe_convert(Ptr{Cvoid}, crulefunc) + end + end +end + +const TRYCONVERT_ERR_ARRAY = Ptr{Cvoid}[] + +PyObject_TryConvert(o::PyPtr, ::Type{T}) where {T} = begin + # First try based only on the type T + # Used mainly by wrapper types for immediate conversion. + r = PyObject_TryConvert__initial(o, T) :: Int + r == 0 || return r + + # Try to find an appropriate conversion based on `T` and the type of `o`. + rules = PyObject_TryConvert_CompiledRules(T) + t = Py_Type(o) + crules = get(rules, t, TRYCONVERT_ERR_ARRAY) + if crules === TRYCONVERT_ERR_ARRAY + crules = PyObject_TryConvert_CompileRule(T, t) + crules === PYERR() && return -1 + rules[t] = crules + end + for crule in crules + r = ccall(crule, Int, (PyPtr,), o) + r == 0 || return r + end + + # Failed to convert + return 0 +end +PyObject_TryConvert(o, ::Type{T}) where {T} = GC.@preserve o PyObject_TryConvert(Base.unsafe_convert(PyPtr, o), T) + +PyObject_TryConvert__initial(o, ::Type{T}) where {T} = 0 + +PyObject_Convert(o::PyPtr, ::Type{T}) where {T} = begin + r = PyObject_TryConvert(o, T) + if r == 0 + PyErr_SetString(PyExc_TypeError(), "Cannot convert this '$(PyType_Name(Py_Type(o)))' to a Julia '$T'") + -1 + elseif r == -1 + -1 + else + 0 + end +end +PyObject_Convert(o, ::Type{T}) where {T} = GC.@preserve o PyObject_Convert(Base.unsafe_convert(PyPtr, o), T) diff --git a/src/cpython/range.jl b/src/cpython/range.jl new file mode 100644 index 00000000..9cad1f1d --- /dev/null +++ b/src/cpython/range.jl @@ -0,0 +1,90 @@ +const PyRange_Type__ref = Ref(PyPtr()) +PyRange_Type() = begin + ptr = PyRange_Type__ref[] + isnull(ptr) || return ptr + bs = PyEval_GetBuiltins() + ptr = PyMapping_GetItemString(bs, "range") + PyRange_Type__ref[] = ptr +end + +PyRange_From(a::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a) +end + +PyRange_From(a::Integer, b::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a, b) +end + +PyRange_From(a::Integer, b::Integer, c::Integer) = begin + t = PyRange_Type() + isnull(t) && return t + PyObject_CallNice(t, a, b, c) +end + +PyRange_From(r::AbstractRange{<:Integer}) = + PyRange_From(first(r), last(r) + sign(step(r)), step(r)) + +steptype(::Type{<:(StepRange{A,B} where {A})}) where {B} = B +steptype(::Type{<:StepRange}) = Any + +PyRange_TryConvertRule_steprange(o, ::Type{S}) where {S<:StepRange} = begin + A = _typeintersect(Integer, eltype(S)) + B = _typeintersect(Integer, steptype(S)) + # get start + ao = PyObject_GetAttrString(o, "start") + isnull(ao) && return -1 + r = PyObject_TryConvert(ao, A) + Py_DecRef(ao) + r == 1 || return r + a = takeresult(A) + # get step + bo = PyObject_GetAttrString(o, "step") + isnull(bo) && return -1 + r = PyObject_TryConvert(bo, B) + Py_DecRef(bo) + r == 1 || return r + b = takeresult(B) + # get stop + co = PyObject_GetAttrString(o, "stop") + isnull(co) && return -1 + r = PyObject_TryConvert(co, A) + Py_DecRef(co) + r == 1 || return r + c = takeresult(A) + # success + a′, c′ = promote(a, c - oftype(c, sign(b))) + putresult(tryconvert(S, StepRange(a′, b, c′))) +end + +PyRange_TryConvertRule_unitrange(o, ::Type{S}) where {S<:UnitRange} = begin + A = _typeintersect(Integer, eltype(S)) + # get step + bo = PyObject_GetAttrString(o, "step") + isnull(bo) && return -1 + r = PyObject_TryConvert(bo, Int) + Py_DecRef(bo) + r == 1 || return r + b = takeresult(Int) + b == 1 || return 0 + # get start + ao = PyObject_GetAttrString(o, "start") + isnull(ao) && return -1 + r = PyObject_TryConvert(ao, A) + Py_DecRef(ao) + r == 1 || return r + a = takeresult(A) + # get stop + co = PyObject_GetAttrString(o, "stop") + isnull(co) && return -1 + r = PyObject_TryConvert(co, A) + Py_DecRef(co) + r == 1 || return r + c = takeresult(A) + # success + a′, c′ = promote(a, c - oftype(c, 1)) + putresult(tryconvert(S, UnitRange(a′, c′))) +end diff --git a/src/cpython/sequence.jl b/src/cpython/sequence.jl new file mode 100644 index 00000000..414ddca5 --- /dev/null +++ b/src/cpython/sequence.jl @@ -0,0 +1,4 @@ +@cdef :PySequence_Length Py_ssize_t (PyPtr,) +@cdef :PySequence_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PySequence_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) +@cdef :PySequence_Contains Cint (PyPtr, PyPtr) diff --git a/src/cpython/set.jl b/src/cpython/set.jl new file mode 100644 index 00000000..aff24111 --- /dev/null +++ b/src/cpython/set.jl @@ -0,0 +1,54 @@ +@cdef :PySet_New PyPtr (PyPtr,) +@cdef :PyFrozenSet_New PyPtr (PyPtr,) +@cdef :PySet_Add Cint (PyPtr, PyPtr) + +const PySet_Type__ref = Ref(PyPtr()) +PySet_Type() = pyglobal(PySet_Type__ref, :PySet_Type) + +PySet_Check(o) = Py_TypeCheckFast(o, PySet_Type()) + +PySet_CheckExact(o) = Py_TypeCheckExact(o, PySet_Type()) + +const PyFrozenSet_Type__ref = Ref(PyPtr()) +PyFrozenSet_Type() = pyglobal(PyFrozenSet_Type__ref, :PyFrozenSet_Type) + +PyFrozenSet_Check(o) = Py_TypeCheckFast(o, PyFrozenSet_Type()) + +PyFrozenSet_CheckExact(o) = Py_TypeCheckExact(o, PyFrozenSet_Type()) + +PyAnySet_Check(o) = PySet_Check(o) || PyFrozenSet_Check(o) + +PyAnySet_CheckExact(o) = PySet_CheckExact(o) || PyFrozenSet_CheckExact(o) + +_PySet_FromIter(r::PyPtr, xs) = begin + try + for x in xs + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(r); return PyPtr()) + err = PySet_Add(r, xo) + Py_DecRef(xo) + ism1(err) && (Py_DecRef(r); return PyPtr()) + end + return r + catch + Py_DecRef(r) + PyErr_SetString(PyExc_Exception(), "Julia error: $err") + return PyPtr() + end +end + +PySet_FromIter(xs) = begin + r = PySet_New(C_NULL) + isnull(r) && return PyPtr() + _PySet_FromIter(r, xs) +end + +PyFrozenSet_FromIter(xs) = begin + r = PyFrozenSet_New(C_NULL) + isnull(r) && return PyPtr() + _PySet_FromIter(r, xs) +end + +PySet_From(x::Union{AbstractSet,AbstractVector,Tuple}) = PySet_FromIter(x) + +PyFrozenSet_From(x::Union{AbstractSet,AbstractVector,Tuple}) = PyFrozenSet_FromIter(x) diff --git a/src/cpython/slice.jl b/src/cpython/slice.jl new file mode 100644 index 00000000..e71ceb9f --- /dev/null +++ b/src/cpython/slice.jl @@ -0,0 +1,17 @@ +@cdef :PySlice_New PyPtr (PyPtr, PyPtr, PyPtr) + +const PySlice_Type__ref = Ref(PyPtr()) +PySlice_Type() = pyglobal(PySlice_Type__ref, :PySlice_Type) + +PySlice_Check(o) = Py_TypeCheck(o, PySlice_Type()) + +PySlice_CheckExact(o) = Py_TypeCheckExact(o, PySlice_Type()) + +### ELLIPSIS + +const Py_Ellipsis__ref = Ref(PyPtr()) +Py_Ellipsis() = pyglobal(Py_Ellipsis__ref, :_Py_EllipsisObject) + +PyEllipsis_Check(o) = Py_Is(o, Py_Ellipsis()) + +PyEllipsis_New() = (o=Py_Ellipsis(); Py_IncRef(o); o) diff --git a/src/cpython/str.jl b/src/cpython/str.jl new file mode 100644 index 00000000..de9c139a --- /dev/null +++ b/src/cpython/str.jl @@ -0,0 +1,53 @@ +@cdef :PyUnicode_DecodeUTF8 PyPtr (Ptr{Cchar}, Py_ssize_t, Ptr{Cvoid}) +@cdef :PyUnicode_AsUTF8String PyPtr (PyPtr,) +@cdef :PyUnicode_InternInPlace Cvoid (Ptr{PyPtr},) + +const PyUnicode_Type__ref = Ref(PyPtr()) +PyUnicode_Type() = pyglobal(PyUnicode_Type__ref, :PyUnicode_Type) + +PyUnicode_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_UNICODE_SUBCLASS) +PyUnicode_CheckExact(o) = Py_TypeCheckExact(o, PyUnicode_Type()) + +PyUnicode_From(s::Union{Vector{Cuchar},Vector{Cchar},String,SubString{String}}) = + PyUnicode_DecodeUTF8(pointer(s), sizeof(s), C_NULL) + +PyUnicode_AsString(o) = begin + b = PyUnicode_AsUTF8String(o) + isnull(b) && return "" + r = PyBytes_AsString(b) + Py_DecRef(b) + r +end + +PyUnicode_AsVector(o, ::Type{T}=UInt8) where {T} = begin + b = PyUnicode_AsUTF8String(o) + isnull(b) && return T[] + r = PyBytes_AsVector(b, T) + Py_DecRef(b) + r +end + +PyUnicode_TryConvertRule_string(o, ::Type{String}) = begin + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + putresult(r) +end + +PyUnicode_TryConvertRule_vector(o, ::Type{Vector{X}}) where {X} = begin + r = PyUnicode_AsVector(o, X) + isempty(r) && PyErr_IsSet() && return -1 + putresult(r) +end + +PyUnicode_TryConvertRule_symbol(o, ::Type{Symbol}) = begin + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + putresult(Symbol(r)) +end + +PyUnicode_TryConvertRule_char(o, ::Type{Char}) = begin + r = PyUnicode_AsString(o) + isempty(r) && PyErr_IsSet() && return -1 + length(r) == 1 || return 0 + putresult(first(r)) +end diff --git a/src/cpython/tuple.jl b/src/cpython/tuple.jl new file mode 100644 index 00000000..9af52363 --- /dev/null +++ b/src/cpython/tuple.jl @@ -0,0 +1,33 @@ +@cdef :PyTuple_New PyPtr (Py_ssize_t,) +@cdef :PyTuple_Size Py_ssize_t (PyPtr,) +@cdef :PyTuple_GetItem PyPtr (PyPtr, Py_ssize_t) +@cdef :PyTuple_SetItem Cint (PyPtr, Py_ssize_t, PyPtr) + +const PyTuple_Type__ref = Ref(PyPtr()) +PyTuple_Type() = pyglobal(PyTuple_Type__ref, :PyTuple_Type) + +PyTuple_Check(o) = Py_TypeCheckFast(o, Py_TPFLAGS_TUPLE_SUBCLASS) + +PyTuple_CheckExact(o) = Py_TypeCheckExact(o, PyTuple_Type()) + +PyTuple_From(x::Union{Tuple,AbstractVector}) = PyTuple_FromIter(x) + +PyTuple_FromIter(xs::Tuple) = begin + t = PyTuple_New(length(xs)) + isnull(t) && return PyPtr() + for (i,x) in enumerate(xs) + xo = PyObject_From(x) + isnull(xo) && (Py_DecRef(t); return PyPtr()) + err = PyTuple_SetItem(t, i-1, xo) # steals xo + ism1(err) && (Py_DecRef(t); return PyPtr()) + end + return t +end + +PyTuple_FromIter(xs) = begin + y = PyList_FromIter(xs) + isnull(y) && return PyPtr() + t = PyList_AsTuple(y) + Py_DecRef(y) + return t +end diff --git a/src/cpython/type.jl b/src/cpython/type.jl new file mode 100644 index 00000000..39125af0 --- /dev/null +++ b/src/cpython/type.jl @@ -0,0 +1,43 @@ +@cdef :PyType_IsSubtype Cint (PyPtr, PyPtr) +@cdef :PyType_Ready Cint (PyPtr,) + +Py_Type(o) = GC.@preserve o UnsafePtr(Base.unsafe_convert(PyPtr, o)).type[!] +Py_TypeCheck(o, t) = PyType_IsSubtype(Py_Type(o), t) != 0 +Py_TypeCheckExact(o, t) = Py_Type(o) == Base.unsafe_convert(PyPtr, t) +Py_TypeCheckFast(o, f) = PyType_IsSubtypeFast(Py_Type(o), f) + +PyType_Flags(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).flags[] +PyType_Name(o) = GC.@preserve o unsafe_string(UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).name[!]) +PyType_MRO(o) = GC.@preserve o UnsafePtr{PyTypeObject}(Base.unsafe_convert(PyPtr, o)).mro[!] + +PyType_IsSubtypeFast(s, f) = PyType_HasFeature(s, f) +PyType_HasFeature(s, f) = !iszero(PyType_Flags(s) & f) + +const PyType_Type__ref = Ref(PyPtr()) +PyType_Type() = pyglobal(PyType_Type__ref, :PyType_Type) + +PyType_Check(o) = Py_TypeCheck(o, Py_TPFLAGS_TYPE_SUBCLASS) + +PyType_CheckExact(o) = Py_TypeCheckExact(o, PyType_Type()) + +PyType_FullName(o) = begin + # get __module__ + mo = PyObject_GetAttrString(o, "__module__") + isnull(mo) && return PYERR() + m = PyUnicode_AsString(mo) + Py_DecRef(mo) + isempty(m) && PyErr_IsSet() && return PYERR() + # get __qualname__ + no = PyObject_GetAttrString(o, "__qualname__") + isnull(no) && return PYERR() + n = PyUnicode_AsString(no) + Py_DecRef(no) + isempty(n) && PyErr_IsSet() && return PYERR() + # done + "$m.$n" +end + +PyType_MROAsVector(o) = begin + mro = PyType_MRO(o) + PyPtr[PyTuple_GetItem(mro, i-1) for i in 1:PyTuple_Size(mro)] +end diff --git a/src/dict.jl b/src/dict.jl deleted file mode 100644 index 05bdaff9..00000000 --- a/src/dict.jl +++ /dev/null @@ -1,34 +0,0 @@ -const pydicttype = pylazyobject(() -> pybuiltins.dict) -export pydicttype - -pyisdict(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_DICT_SUBCLASS) -export pyisdict - -pydict(args...; opts...) = pydicttype(args...; opts...) -pydict() = check(C.PyDict_New()) -pydict(o::Union{AbstractDict, Base.Iterators.Pairs, AbstractArray{<:Pair}}) = pydict_fromiter(o) -pydict(o::NamedTuple) = pydict_fromiter(pairs(o)) -export pydict - -function pydict_fromiter(kvs) - d = pydict() - for (k,v) in kvs - vo = pyobject(v) - if k isa AbstractString - check(C.PyDict_SetItemString(d, k, vo)) - else - check(C.PyDict_SetItem(d, pyobject(k), vo)) - end - end - return d -end - -function pydict_fromstringiter(kvs) - d = pydict() - for (k,v) in kvs - ko = k isa AbstractString ? k : k isa Symbol ? String(k) : error("only string and symbols allowed") - vo = pyobject(v) - check(C.PyDict_SetItemString(d, ko, vo)) - end - return d -end diff --git a/src/error.jl b/src/error.jl deleted file mode 100644 index 640a993f..00000000 --- a/src/error.jl +++ /dev/null @@ -1,136 +0,0 @@ -check(::Type{CPyPtr}, o::CPyPtr, ambig=false) = (o == C_NULL && (ambig ? pyerrcheck() : pythrow()); o) -check(::Type{T}, o::T, ambig=false) where {T<:Number} = (o == (zero(T)-one(T)) && (ambig ? pyerrcheck() : pythrow()); o) -check(::Type{Nothing}, o::Cvoid, ambig=false) = ambig ? pyerrcheck() : nothing -check(::Type{PyObject}, o::CPyPtr, ambig=false) = pynewobject(check(CPyPtr, o, ambig)) -check(::Type{Bool}, o::Cint, ambig=false) = !iszero(check(Cint, o, ambig)) -check(::Type{Nothing}, o::Cint, ambig=false) = (check(Cint, o, ambig); nothing) -check(o::CPyPtr, ambig=false) = check(PyObject, o, ambig) -check(o::T, ambig=false) where {T<:Number} = check(T, o, ambig) -check(o::Cvoid, ambig=false) = check(Nothing, o, ambig) - - -pyerroccurred() = C.PyErr_Occurred() != C_NULL - -function pyerroccurred(t::PyObject) - e = C.PyErr_Occurred() - e != C_NULL && !iszero(C.PyErr_GivenExceptionMatches(e, t)) -end - -pyerrclear() = C.PyErr_Clear() - -pyerrcheck() = pyerroccurred() ? pythrow() : nothing - -pyerrset(t::PyObject) = C.PyErr_SetNone(t) -pyerrset(t::PyObject, x::AbstractString) = C.PyErr_SetString(t, x) -pyerrset(t::PyObject, x::PyObject) = C.PyErr_SetObject(t, x) - -pyerrmatches(e::PyObject, t::PyObject) = !iszero(C.PyErr_GivenExceptionMatches(e, t)) - -struct PythonRuntimeError <: Exception - t :: PyObject - v :: PyObject - b :: PyObject -end - -function pyerrfetch(normalize::Bool=false) - t = Ref{CPyPtr}() - v = Ref{CPyPtr}() - b = Ref{CPyPtr}() - C.PyErr_Fetch(t, v, b) - normalize && C.PyErr_NormalizeException(t, v, b) - to = t[]==C_NULL ? pynone : pynewobject(t[]) - vo = v[]==C_NULL ? pynone : pynewobject(v[]) - bo = b[]==C_NULL ? pynone : pynewobject(b[]) - (to, vo, bo) -end - -pyerrrestore(t::PyObject, v::PyObject, b::PyObject) = - C.PyErr_Restore(pyincref!(t), pyincref!(v), pyincref!(b)) - -pyerrrestore(err::PythonRuntimeError) = pyerrrestore(err.t, err.v, err.b) - -pythrow() = throw(PythonRuntimeError(pyerrfetch(true)...)) - -pythrow(v::PyObject) = - throw(PythonRuntimeError(pytype(v), v, pynone)) - -function Base.showerror(io::IO, e::PythonRuntimeError) - if pyisnone(e.t) - print(io, "Python: mysterious error (no error was actually set)") - return - end - - if CONFIG.sysautolasttraceback - try - pysysmodule.last_type = e.t - pysysmodule.last_value = e.v - pysysmodule.last_traceback = e.b - catch err - print(io, "") - end - end - - # if this is a Julia exception then recursively print it and its stacktrace - if pyerrmatches(e.t, pyjlexception) - try - jp = pyjlgetvalue(e.v.args[0]) - if jp !== nothing - je, jb = jp - print(io, "Python: Julia: ") - showerror(io, je) - if jb === nothing - println(io) - print(io, "Stacktrace: none") - else - io2 = IOBuffer() - Base.show_backtrace(IOContext(io2, :color=>true, :displaysize=>displaysize(io)), jb) - printstyled(io, String(take!(io2))) - end - end - if pyisnone(e.b) - println(io) - printstyled(io, "Python stacktrace: none") - return - else - @goto pystacktrace - end - catch err - println(io, "") - end - end - - # otherwise, print the Python exception - print(io, "Python: ") - try - print(io, e.t.__name__) - catch err - print(io, "") - end - if !pyisnone(e.v) - print(io, ": ") - try - print(io, e.v) - catch err - print(io, "") - end - end - if !pyisnone(e.b) - @label pystacktrace - println(io) - printstyled(io, "Python stacktrace:") - try - fs = pytracebackmodule.extract_tb(e.b) - nfs = pylen(fs) - for i in 1:nfs - println(io) - f = fs[nfs-i] - printstyled(io, " [", i, "] ", ) - printstyled(io, f.name, bold=true) - printstyled(io, " at ") - printstyled(io, f.filename, ":", f.lineno, bold=true) - end - catch err - print(io, "") - end - end -end diff --git a/src/eval.jl b/src/eval.jl index 0e28a9bc..2171f93f 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -1,137 +1,203 @@ -const SCOPES = Dict{Any,PyObject}() -const COMPILECACHE = Dict{Tuple{String,String,String},PyObject}() - -scope(s) = get!(pydict, SCOPES, s) -scope(s::PyObject) = s +pyeval_filename(src) = isfile(string(src.file)) ? "$(src.file):$(src.line)" : "julia:$(src.file):$(src.line)" + +pyeval_macro(filename, mode, codearg, args...) = begin + # unwrap rettype + if codearg isa Expr && codearg.head == :(::) + rettypearg = codearg.args[end] + codearg = codearg.args[1] + else + rettypearg = mode == :eval ? PyObject : Nothing + end + # extract the code + code = codearg.args[end] + # parse the remaining arguments - the first one is optionally the locals, the rest are var=value pairs to include in the locals + kvs = [] + locals = nothing + for (i,arg) in enumerate(args) + if arg isa Expr && arg.head == :(=) && arg.args[1] isa Symbol + push!(kvs, arg.args[1] => arg.args[2]) + elseif i == 1 + locals = arg + else + error() + end + end + # find any interpolations + i = firstindex(code) + codechunks = String[] + interps = [] + chunk = "" + while true + j = findnext('$', code, i) + if j === nothing + push!(codechunks, chunk * code[i:end]) + break + elseif checkbounds(Bool, code, j+1) && code[j+1] == '$' + chunk *= code[i:j] + i = j+2 + else + push!(codechunks, chunk * code[i:j-1]) + chunk = "" + ex, i = Meta.parse(code, j+1, greedy=false) + push!(interps, ex) + end + end + # identify which are LHS and which are RHS + # currently only the pattern "^ $(...) =" is recognized as a LHS + # TODO: multiple assignment + # TODO: mutating assignment + islhs = [((match(r"[\n;=]\s*$", codechunks[i])!==nothing) || (i==1 && match(r"^\s*$", codechunks[i])!==nothing)) && match(r"^\s*=($|[^=])", codechunks[i+1])!==nothing for (i,ex) in enumerate(interps)] + # do the interpolation + intvars = ["_jl_interp_$(i)_" for i in 1:length(interps)] + for (k,v,lhs) in zip(intvars, interps, islhs) + lhs || push!(kvs, Symbol(k) => v) + end + newcode = join([c * (checkbounds(Bool, intvars, i) ? islhs[i] ? intvars[i] : "($(intvars[i]))" : "") for (i,c) in enumerate(codechunks)]) + # for LHS interpolation, extract out type annotations + if any(islhs) + mode == :exec || error("interpolation on LHS only allowed in exec mode") + end + if mode == :exec + outkvts = [(ex isa Expr && ex.head == :(::)) ? (esc(ex.args[1]), v, esc(ex.args[2])) : (esc(ex), v, PyObject) for (ex,v,lhs) in zip(interps,intvars,islhs) if lhs] + end + # make the code object + co = PyCode(newcode, filename, mode) + # go + freelocals = locals === nothing ? :(C.Py_DecRef(lptr)) : :(GC.@preserve locals nothing) + ret = quote + let + # evaluate the inputs (so any errors are thrown before we have object references) + $([:($(Symbol(:input,i)) = $(esc(v))) for (i,(k,v)) in enumerate(kvs)]...) + # get the code pointer + cptr = checknull(pyptr($co)) + # get the globals pointer + globals = $(esc(:pyglobals)) + gptr = checknull(pyptr(globals)) + # ensure globals includes builtins + if !globals.hasbuiltins + if C.PyMapping_HasKeyString(gptr, "__builtins__") == 0 + err = C.PyMapping_SetItemString(gptr, "__builtins__", C.PyEval_GetBuiltins()) + ism1(err) && pythrow() + end + globals.hasbuiltins = true + end + # get locals (ALLOCATES lptr if locals===nothing) + $(locals === nothing ? :(lptr = checknull(C.PyDict_New())) : :(locals = $(esc(locals)); lptr = checknull(pyptr(locals)))) + # insert extra locals + $([:(let; vo=C.PyObject_From($(Symbol(:input,i))); isnull(vo) && ($freelocals; pythrow()); err=C.PyObject_SetItem(lptr, $(PyInternedString(string(k))), vo); C.Py_DecRef(vo); ism1(err) && ($freelocals; pythrow()); end) for (i,(k,v)) in enumerate(kvs)]...) + # Call eval (ALLOCATES rptr) + rptr = GC.@preserve globals C.PyEval_EvalCode(cptr, gptr, lptr) + isnull(rptr) && ($freelocals; pythrow()) + # extract values + $( + if mode == :eval + quote + $freelocals + res = C.PyObject_Convert(rptr, $(esc(rettypearg))) + C.Py_DecRef(rptr) + ism1(res) && pythrow() + C.takeresult($(esc(rettypearg))) + end + elseif mode == :exec + quote + C.Py_DecRef(rptr) + $((quote + $(Symbol(:output,i)) = let + xo = C.PyObject_GetItem(lptr, $(PyInternedString(v))) + isnull(xo) && ($freelocals; pythrow()) + res = C.PyObject_Convert(xo, $t) + C.Py_DecRef(xo) + ism1(res) && ($freelocals; pythrow()) + C.takeresult($t) + end + end for (i,(_,v,t)) in enumerate(outkvts))...) + $freelocals + ($((Symbol(:output,i) for i in 1:length(outkvts))...),) + end + else + error() + end + ) + end + end + if mode == :exec + ret = quote + ($((k for (k,_,_) in outkvts)...),) = $ret + nothing + end + end + ret +end """ - pyeval(src, scope, locals=nothing) + @py `...` [locals] [var=val, ...] -Evaluate Python expression `src` in the context of the given `scope` and `locals`. Return the value. +Executes the given Python code. -If the `scope` is a Python object, it must be a `dict` to use as globals. +Julia values can be interpolated using the usual `\$(...)` syntax. -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyeval(src, @__MODULE__)` evaluates in a scope unique to the current module. -""" -pyeval(src, globals, locals=nothing) = pyevalfunc(src, scope(globals), locals) +Additionally, assignment to interpolations is supported: e.g. `\$(x::T) = ...` will convert the right hand side to a `T` and assign it to `x`. +- Currently only single assignment is supported. Multiple assignment (`\$x, \$y = ...`) or mutating assignment (`\$x += ...`) will not be recognized. +- What actually happens is that the assignment is to a temporary Python variable, which is then read when execution successfully finishes. + Hence if an exception occurs, no assignments will happen. +The globals are `pyglobals`. +The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. """ - pyexec(src, scope, locals=nothing) +macro py(args...) + pyeval_macro(pyeval_filename(__source__), :exec, args...) +end +export @py -Execute Python expression `src` in the context of the given `scope` and `locals`. +""" + @pyg `...` [var=val, ...] -If the `scope` is a Python object, it must be a `dict` to use as globals. +Executes the given Python code in the global scope. -Otherwise, a globals dict is created and reused for each unique `scope`. For example `pyexec(src, @__MODULE__)` executes in a scope unique to the current module. +This is simply shorthand for ```@py `...` pyglobals ``` (see [`@py`](@ref)). """ -pyexec(src, globals, locals=nothing) = (pyexecfunc(src, scope(globals), locals); nothing) +macro pyg(code, args...) + :(@py $code $(esc(:pyglobals)) $(args...)) +end +export @pyg """ - pyevaldb(src, scope, locals=nothing) + @pyv `...`[::rettype] [locals] [var=val, ...] -Same as `pyeval(src, scope, locals)` but evaluated inside a `pdb` debugger. -""" -pyevaldb(src, globals, locals=nothing) = pypdbmodule.runeval(src, scope(globals), locals) +Evaluate the given Python code. -""" - pyexecdb(src, scope, locals=nothing) +Julia values can be interpolated using the usual `\$(...)` syntax. -Same as `pyexec(src, scope, locals)` but evaluated inside a `pdb` debugger. +The globals are `pyglobals`. +The locals are `locals`, if given, otherwise a temporary scope is created. Extra values to be interted into the scope can be given with extra `var=val` arguments. + +The result is converted to a `rettype`, which defaults to `PyObject`. """ -pyexecdb(src, globals, locals=nothing) = (pypdbmodule.run(src, scope(globals), locals); nothing) +macro pyv(args...) + pyeval_macro(pyeval_filename(__source__), :eval, args...) +end +export @pyv """ - py"...."[flags] + py`...` :: PyCode -Evaluate (`v`) or execute (`x`) the given Python source code. +A Python code object in "exec" mode which is compiled only once. -Julia values may be interpolated into the source code with `\$` syntax. For a literal `\$`, enter `\$\$`. +Suitable for using as the `code` argument to `pyeval`. +""" +macro py_cmd(code::String) + PyCode(code, pyeval_filename(__source__), :exec) +end +export @py_cmd -Execution occurs in a global scope unique to the current module. +""" + pyv`...` :: PyCode -The flags can be any combination of the following characters: -- `v` (evaluate): Evaluate a single expression and return its value. -- `x` (execute): Execute the code and return `nothing`. -- `g` (globals): Return the dict of globals for the scope instead. -- `l` (locals): Perform the computation in a new local scope and return that scope. -- `c` (compile): Cache a compiled version of the source and re-use it each time, for speed. -- `d` (debug): Run inside a `pdb` debugger. +A Python code object in "eval" mode which is compiled only once. -If neither `v` nor `x` is specified and the code is a single line then `v` (evaluate) is assumed, otherwise `x` (execute). +Suitable for using as the `code` argument to `pyexec`. """ -macro py_str(src::String, flags::String="") - # parse the flags - exec = '\n' in src - retglobals = false - retlocals = false - lazy = false - compile = false - debug = false - for f in flags - if f == 'x' - exec = true - elseif f == 'v' - exec = false - elseif f == 'g' - retglobals = true - elseif f == 'l' - retlocals = true - # elseif f == 'z' - # lazy = true - elseif f == 'c' - compile = true - elseif f == 'd' - debug = true - else - error("invalid flags: `py\"...\"$flags`") - end - end - # parse src for $-interpolations - chunks = String[] - interps = [] - i = firstindex(src) - while true - j = findnext('$', src, i) - if j === nothing - push!(chunks, src[i:end]) - break - else - push!(chunks, src[i:prevind(src,j)]) - end - if checkbounds(Bool, src, j+1) && src[j+1] == '$' - push!(chunks, "\$") - i = j+2 - else - ex, i = Meta.parse(src, j+1, greedy=false) - var = "_jl_interp_$(length(interps)+1)_" - push!(interps, var => ex) - push!(chunks, "($var)") - end - end - newsrcstr = newsrc = join(chunks) - # compile the code so there is string information - cfile = "julia:$(__source__.file):$(__source__.line)" - cmode = exec ? "exec" : "eval" - newsrc = :(pycompile($newsrcstr, $cfile, $cmode)) - if compile - # compile the code lazily (so the python parser is only invoked once) - # Julia crashes if you try to put pylazyobject(()->pycompile(...)) directly in the syntax tree. Is this a bug?? - # Instead, we use a run-time lookup table. - newsrc = :(get!(()->$newsrc, COMPILECACHE, ($newsrcstr, $cfile, $cmode))) - end - # make the expression - ex = :(let - globals = scope(@__MODULE__) - locals = $(retlocals ? :(pydict()) : :globals) - $([:(check(C.PyDict_SetItemString(globals, $k, pyobject($(esc(v)))))) for (k,v) in interps]...) - result = $(debug ? (exec ? :pyexecdb : :pyevaldb) : (exec ? :pyexec : :pyeval))($newsrc, globals, locals) - $([:(check(C.PyDict_DelItemString(globals, $k))) for (k,v) in interps]...) - $(retlocals ? :locals : retglobals ? :globals : exec ? nothing : :result) - end) - if lazy - # wrap as a lazy object - ex = :(pylazyobject(() -> $ex)) - end - ex +macro pyv_cmd(code::String) + PyCode(code, pyeval_filename(__source__), :eval) end -export @py_str +export @pyv_cmd diff --git a/src/float.jl b/src/float.jl deleted file mode 100644 index 0dce1b6f..00000000 --- a/src/float.jl +++ /dev/null @@ -1,22 +0,0 @@ -const pyfloattype = pylazyobject(() -> pybuiltins.float) -export pyfloattype - -pyfloat(args...; opts...) = pyfloattype(args...; opts...) -pyfloat(x::Real) = check(C.PyFloat_FromDouble(x)) -export pyfloat - -pyisfloat(o::PyObject) = pytypecheck(o, pyfloattype) -export pyisfloat - -function pyfloat_tryconvert(::Type{T}, o::PyObject) where {T} - x = check(C.PyFloat_AsDouble(o), true) - if (S = _typeintersect(T, Cdouble)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, AbstractFloat)) != Union{} - tryconvert(S, x) - elseif (S = _typeintersect(T, Real)) != Union{} - tryconvert(S, x) - else - tryconvert(T, x) - end -end diff --git a/src/fraction.jl b/src/fraction.jl deleted file mode 100644 index c28e31e6..00000000 --- a/src/fraction.jl +++ /dev/null @@ -1,33 +0,0 @@ -const pyfractiontype = pylazyobject(() -> pyfractionsmodule.Fraction) -export pyfractiontype - -pyfraction(args...; opts...) = pyfractiontype(args...; opts...) -pyfraction(x::Rational) = pyfraction(numerator(x), denominator(x)) -export pyfraction - -pyisfraction(o::PyObject) = pytypecheck(o, pyfractiontype) -export pyisfraction - -function pyfraction_tryconvert(::Type{T}, o::PyObject) where {T<:Rational} - tryconvert(T, pyfraction_tryconvert(Rational{BigInt}, o)) -end - -function pyfraction_tryconvert(::Type{Rational{T}}, o::PyObject) where {T<:Integer} - x = pyint_tryconvert(T, o.numerator) - x === PyConvertFail() && return x - y = pyint_tryconvert(T, o.denominator) - y === PyConvertFail() && return y - Rational{T}(x, y) -end - -function pyfraction_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, Rational)) != Union{} - pyfraction_tryconvert(S, o) - elseif (S = _typeintersect(T, Integer)) != Union{} - o.denominator == pyint(1) ? pyint_tryconvert(S, o.numerator) : PyConvertFail() - elseif (S = _typeintersect(T, Number)) != Union{} - tryconvert(S, pyfraction_tryconvert(Rational{BigInt}, o)) - else - tryconvert(T, pyfraction_tryconvert(Rational{BigInt}, o)) - end -end diff --git a/src/gil.jl b/src/gil.jl index 01785440..563a058d 100644 --- a/src/gil.jl +++ b/src/gil.jl @@ -1,5 +1,14 @@ -function with_gil(f) - if CONFIG.isembedded +""" + with_gil(f, [c=true]) + +Compute `f()` with the GIL enabled. + +This may need a `try-finally` block to ensure the GIL is released again. If you know that `f` cannot throw, pass `c=false` to avoid this overhead. +""" +function with_gil(f, c::Bool=true) + if !CONFIG.isembedded + f() + elseif c g = C.PyGILState_Ensure() try f() @@ -7,6 +16,9 @@ function with_gil(f) C.PyGILState_Release(g) end else - f() + g = C.PyGILState_Ensure() + r = f() + C.PyGILState_Release(g) + r end end diff --git a/src/gui.jl b/src/gui.jl index c98d1d78..459d70ba 100644 --- a/src/gui.jl +++ b/src/gui.jl @@ -10,9 +10,9 @@ one when using this package. If `CONFIG.qtfix` is true, then this is run automatically before `PyQt4`, `PyQt5`, `PySide` or `PySide2` are imported. """ function fix_qt_plugin_path() - e = pyosmodule.environ - "QT_PLUGIN_PATH" in e && return false CONFIG.exepath === nothing && return false + e = pyosmodule().environ + "QT_PLUGIN_PATH" in e && return false qtconf = joinpath(dirname(CONFIG.exepath), "qt.conf") isfile(qtconf) || return false for line in eachline(qtconf) @@ -63,83 +63,83 @@ function event_loop_off(g::Symbol) return end -function event_loop_on(g::Symbol; interval::Real=40e-3, fix::Bool=false) - # check if already running - if haskey(EVENT_LOOPS, g) - return g => EVENT_LOOPS[g] - end - # start a new event loop - if g in (:pyqt4, :pyqt5, :pyside, :pyside2) - fix && fix_qt_plugin_path() - modname = - g == :pyqt4 ? "PyQt4" : - g == :pyqt5 ? "PyQt5" : - g == :pyside ? "PySide" : - g == :pyside2 ? "PySide2" : error() - mod = pyimport("$modname.QtCore") - instance = mod.QCoreApplication.instance - AllEvents = mod.QEventLoop.AllEvents - processEvents = mod.QCoreApplication.processEvents - maxtime = pyobject(1000*interval) - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - app = instance() - if !pyisnone(app) - app._in_event_loop = true - processEvents(AllEvents, maxtime) - end - end - elseif g in (:gtk, :gtk3) - if g == :gtk3 - gi = pyimport("gi") - if pyisnone(gi.get_required_version("Gtk")) - gi.require_version("Gtk", "3.0") - end - end - mod = pyimport(g==:gtk ? "gtk" : g==:gtk3 ? "gi.repository.Gtk" : error()) - events_pending = mod.events_pending - main_iteration = mod.main_iteration - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - while pytruth(events_pending()) - main_iteration() - end - end - elseif g in (:wx,) - mod = pyimport("wx") - GetApp = mod.GetApp - EventLoop = mod.EventLoop - EventLoopActivator = mod.EventLoopActivator - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - app = GetApp() - if !pyisnone(app) - app._in_event_loop = true - evtloop = EventLoop() - ea = EventLoopActivator(evtloop) - Pending = evtloop.Pending - Dispatch = evtloop.Dispatch - while pytruth(Pending()) - Dispatch() - end - finalize(ea) # deactivate event loop - app.ProcessIdle() - end - end - elseif g in (:tkinter,) - mod = pyimport("tkinter") - _tkinter = pyimport("_tkinter") - flag = _tkinter.ALL_EVENTS | _tkinter.DONT_WAIT - root = PyObject(pynone) - EVENT_LOOPS[g] = Timer(0; interval=interval) do t - new_root = mod._default_root - if !pyisnone(new_root) - root = new_root - end - if !pyisnone(root) - while pytruth(root.dooneevent(flag)) - end - end - end - else - error("invalid gui: $(repr(g))") - end - g => EVENT_LOOPS[g] -end +# function event_loop_on(g::Symbol; interval::Real=40e-3, fix::Bool=false) +# # check if already running +# if haskey(EVENT_LOOPS, g) +# return g => EVENT_LOOPS[g] +# end +# # start a new event loop +# if g in (:pyqt4, :pyqt5, :pyside, :pyside2) +# fix && fix_qt_plugin_path() +# modname = +# g == :pyqt4 ? "PyQt4" : +# g == :pyqt5 ? "PyQt5" : +# g == :pyside ? "PySide" : +# g == :pyside2 ? "PySide2" : error() +# mod = pyimport("$modname.QtCore") +# instance = mod.QCoreApplication.instance +# AllEvents = mod.QEventLoop.AllEvents +# processEvents = mod.QCoreApplication.processEvents +# maxtime = pyobject(1000*interval) +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# app = instance() +# if !pyisnone(app) +# app._in_event_loop = true +# processEvents(AllEvents, maxtime) +# end +# end +# elseif g in (:gtk, :gtk3) +# if g == :gtk3 +# gi = pyimport("gi") +# if pyisnone(gi.get_required_version("Gtk")) +# gi.require_version("Gtk", "3.0") +# end +# end +# mod = pyimport(g==:gtk ? "gtk" : g==:gtk3 ? "gi.repository.Gtk" : error()) +# events_pending = mod.events_pending +# main_iteration = mod.main_iteration +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# while pytruth(events_pending()) +# main_iteration() +# end +# end +# elseif g in (:wx,) +# mod = pyimport("wx") +# GetApp = mod.GetApp +# EventLoop = mod.EventLoop +# EventLoopActivator = mod.EventLoopActivator +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# app = GetApp() +# if !pyisnone(app) +# app._in_event_loop = true +# evtloop = EventLoop() +# ea = EventLoopActivator(evtloop) +# Pending = evtloop.Pending +# Dispatch = evtloop.Dispatch +# while pytruth(Pending()) +# Dispatch() +# end +# finalize(ea) # deactivate event loop +# app.ProcessIdle() +# end +# end +# elseif g in (:tkinter,) +# mod = pyimport("tkinter") +# _tkinter = pyimport("_tkinter") +# flag = _tkinter.ALL_EVENTS | _tkinter.DONT_WAIT +# root = PyObject(pynone) +# EVENT_LOOPS[g] = Timer(0; interval=interval) do t +# new_root = mod._default_root +# if !pyisnone(new_root) +# root = new_root +# end +# if !pyisnone(root) +# while pytruth(root.dooneevent(flag)) +# end +# end +# end +# else +# error("invalid gui: $(repr(g))") +# end +# g => EVENT_LOOPS[g] +# end diff --git a/src/import.jl b/src/import.jl deleted file mode 100644 index 06d0263d..00000000 --- a/src/import.jl +++ /dev/null @@ -1,34 +0,0 @@ -""" - pyimport(m, ...; [condapkg], [condachannel]) - pyimport(m => k, ...) - -Import and return the module `m`. - -If additionally `k` is given, then instead returns this attribute from `m`. If it is a tuple, a tuple of attributes is returned. - -If two or more arguments are given, they are all imported and returned as a tuple. - -If there is no such module and `condapkg` is given, then the given package will be automatically installed. Additionally, `condachannel` specifies the channel. -""" -function pyimport(m; condapkg::Union{Nothing,AbstractString}=nothing, condachannel::AbstractString="") - if condapkg === nothing || !CONFIG.isconda - m isa AbstractString ? check(C.PyImport_ImportModule(m)) : check(C.PyImport_Import(pyobject(m))) - else - try - pyimport(m) - catch err - if err isa PythonRuntimeError && pyissubclass(err.t, pymodulenotfounderror) - Conda.add(condapkg, CONFIG.condaenv; channel=condachannel) - pyimport(m) - else - rethrow() - end - end - end -end -function pyimport(x::Pair; opts...) - m = pyimport(x[1]; opts...) - x[2] isa Tuple ? map(k->pygetattr(m, k), x[2]) : pygetattr(m, x[2]) -end -pyimport(m1, m2, ms...) = (pyimport(m1), pyimport(m2), map(pyimport, ms)...) -export pyimport diff --git a/src/init.jl b/src/init.jl index f7eb725f..3c1ec953 100644 --- a/src/init.jl +++ b/src/init.jl @@ -1,9 +1,3 @@ -function python_cmd(args) - env = copy(ENV) - env["PYTHONIOENCODING"] = "UTF-8" - setenv(`$(CONFIG.exepath) $args`, env) -end - function __init__() # Check if libpython is already loaded (i.e. if the Julia interpreter was started from a Python process) CONFIG.isembedded = haskey(ENV, "PYTHONJL_LIBPTR") @@ -43,6 +37,13 @@ function __init__() """) end + # For calling Python with UTF-8 IO + function python_cmd(args) + env = copy(ENV) + env["PYTHONIOENCODING"] = "UTF-8" + setenv(`$(CONFIG.exepath) $args`, env) + end + # Find Python library libpath = something(CONFIG.libpath, get(ENV, "PYTHONJL_LIB", nothing), Some(nothing)) if libpath !== nothing @@ -116,41 +117,74 @@ function __init__() check(C.Py_AtExit(@cfunction(()->(CONFIG.isinitialized = false; nothing), Cvoid, ()))) atexit() do CONFIG.isinitialized = false - check(C.Py_FinalizeEx()) + checkm1(C.Py_FinalizeEx()) end + end + end + C.PyObject_TryConvert_AddRules("builtins.object", [ + (PyObject, CTryConvertRule_wrapref, -100), + (PyRef, CTryConvertRule_wrapref, -200), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Sequence", [ + (PyList, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Set", [ + (PySet, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("collections.abc.Mapping", [ + (PyDict, CTryConvertRule_wrapref, 100), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), + (PyBuffer, CTryConvertRule_wrapref, -200), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 200), + ]) + C.PyObject_TryConvert_AddRules("", [ + (PyArray, CTryConvertRule_trywrapref, 0), + ]) + + with_gil() do + + @pyg `import sys, os` + + if !CONFIG.isembedded + @py ``` # Some modules expect sys.argv to be set - pysysmodule.argv = pylist([""; ARGS]) + # TODO: Append ARGS + sys.argv = [""] # Some modules test for interactivity by checking if sys.ps1 exists - if isinteractive() && !pyhasattr(pysysmodule, "ps1") - pysysmodule.ps1 = ">>> " - end + if $(isinteractive()) and not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + ``` end - end - - with_gil() do # Is this the same Python as in Conda? if !CONFIG.isconda && haskey(ENV, "CONDA_PREFIX") && isdir(ENV["CONDA_PREFIX"]) && haskey(ENV, "CONDA_PYTHON_EXE") && isfile(ENV["CONDA_PYTHON_EXE"]) && - realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? pyconvert(String, pysysmodule.executable) : CONFIG.exepath) + realpath(ENV["CONDA_PYTHON_EXE"]) == realpath(CONFIG.exepath===nothing ? @pyv(`sys.executable`::String) : CONFIG.exepath) CONFIG.isconda = true CONFIG.condaenv = ENV["CONDA_PREFIX"] - CONFIG.exepath === nothing && (CONFIG.exepath = pyconvert(String, pysysmodule.executable)) + CONFIG.exepath === nothing && (CONFIG.exepath = @pyv(`sys.executable`::String)) end # Get the python version - CONFIG.version = let (a,b,c,d,e) = pyconvert(Tuple{Int,Int,Int,String,Int}, pysysmodule.version_info) + CONFIG.version = let (a,b,c,d,e) = @pyv(`sys.version_info`::Tuple{Int,Int,Int,String,Int}) VersionNumber(a, b, c, (d,), (e,)) end - v"2" < CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version.major).$(CONFIG.version.minor) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") + v"3" ≤ CONFIG.version < v"4" || error("Only Python 3 is supported, this is Python $(CONFIG.version) at $(CONFIG.exepath===nothing ? "unknown location" : CONFIG.exepath).") # EXPERIMENTAL: hooks to perform actions when certain modules are loaded if !CONFIG.isembedded - py""" + @py ``` import sys class JuliaCompatHooks: def __init__(self): @@ -171,18 +205,16 @@ function __init__() sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) # Before Qt is loaded, fix the path used to look up its plugins - qtfix_hook = $(pyjlfunction(() -> if CONFIG.qtfix; fix_qt_plugin_path(); nothing; end)) + qtfix_hook = $(() -> (CONFIG.qtfix && fix_qt_plugin_path(); nothing)) JULIA_COMPAT_HOOKS.add_hook("PyQt4", qtfix_hook) JULIA_COMPAT_HOOKS.add_hook("PyQt5", qtfix_hook) JULIA_COMPAT_HOOKS.add_hook("PySide", qtfix_hook) JULIA_COMPAT_HOOKS.add_hook("PySide2", qtfix_hook) - """ + ``` @require IJulia="7073ff75-c697-5162-941a-fcdaad2a7d2a" begin IJulia.push_postexecute_hook() do - if CONFIG.pyplotautoshow && "matplotlib.pyplot" in pysysmodule.modules - pyplotshow() - end + CONFIG.pyplotautoshow && pyplotshow() end end end diff --git a/src/int.jl b/src/int.jl deleted file mode 100644 index c5f1af79..00000000 --- a/src/int.jl +++ /dev/null @@ -1,67 +0,0 @@ -const pyinttype = pylazyobject(() -> pybuiltins.int) -export pyinttype - -pyint(args...; opts...) = pyinttype(args...; opts...) -pyint(x::Integer) = - if typemin(Clonglong) ≤ x ≤ typemax(Clonglong) - check(C.PyLong_FromLongLong(x)) - else - # TODO: it's probably faster to do this in base 16 - check(C.PyLong_FromString(string(convert(BigInt, x)), C_NULL, 10)) - end -pyint(x::Unsigned) = - if x ≤ typemax(Culonglong) - check(C.PyLong_FromUnsignedLongLong(x)) - else - pyint(BigInt(x)) - end -export pyint - -pyisint(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_LONG_SUBCLASS) -export pyisint - -function pyint_tryconvert(::Type{T}, o::PyObject) where {T} - if BigInt <: T - # if it fits in a longlong, use that - rl = C.PyLong_AsLongLong(o) - if rl != -1 || !pyerroccurred() - # NOTE: In this case we return an Int if possible, since a lot of Julia functions take an Int but not a BigInt - # This is the only major exception to the rule that the Python type determines the Julia type. - return (Int <: T && typemin(Int) ≤ rl ≤ typemax(Int)) ? Int(rl) : BigInt(rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - else - pyerrclear() - return parse(BigInt, pystr(String, o)) - end - elseif (S = _typeintersect(T, Integer)) != Union{} - if S <: Unsigned - # if it fits in a ulonglong, use that - rl = C.PyLong_AsUnsignedLongLong(o) - if rl != zero(Culonglong)-one(Culonglong) || !pyerroccurred() - return tryconvert(S, rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - elseif S in (UInt8, UInt16, UInt32, UInt64, UInt128) && sizeof(S) ≤ sizeof(Culonglong) - pyerrclear() - return PyConvertFail() - end - else - # if it fits in a longlong, use that - rl = C.PyLong_AsLongLong(o) - if rl != -1 || !pyerroccurred() - return tryconvert(S, rl) - elseif !pyerroccurred(pyoverflowerror) - pythrow() - elseif S in (Int8, Int16, Int32, Int64, Int128) && sizeof(S) ≤ sizeof(Clonglong) - pyerrclear() - return PyConvertFail() - end - end - # last resort: print to a string - pyerrclear() - return tryconvert(S, parse(BigInt, pystr(String, o))) - else - tryconvert(T, pyint_tryconvert(BigInt, o)) - end -end diff --git a/src/io.jl b/src/io.jl deleted file mode 100644 index ee1a6306..00000000 --- a/src/io.jl +++ /dev/null @@ -1,5 +0,0 @@ -const pyiounsupportedoperation = pylazyobject(() -> pyimport("io").UnsupportedOperation) - -pybufferedio(io::IO) = pyjlbufferedio(io) -pytextio(io::IO) = pyjltextio(io) -export pybufferedio, pytextio diff --git a/src/julia.jl b/src/julia.jl index b915cb0d..f8c2c71b 100644 --- a/src/julia.jl +++ b/src/julia.jl @@ -1,1172 +1,37 @@ -const PYJLGCCACHE = Dict{CPyPtr, Any}() +pyjlbasetype(::Type{T}) where {T} = checknullconvert(T, C.PyJuliaBaseValue_Type()) +pyjlbasetype() = pyjlbasetype(PyObject) -function cpycatch(f, ::Type{T}=CPyPtr) where {T} - try - cpyreturn(T, f()) - catch err - if err isa PythonRuntimeError - # We restore Python errors. - # TODO: Is this the right behaviour? - pyerrrestore(err) - else - # Other (Julia) errors are raised as a JuliaException - bt = catch_backtrace() - val = try - pyjlbase((err, bt)) - catch - pynone - end - pyerrset(pyjlexception, val) - end - cpyerrval(T) - end -end +pyjlrawtype(::Type{T}) where {T} = checknullconvert(T, C.PyJuliaRawValue_Type()) +pyjlrawtype() = pyjlrawtype(PyObject) -cpyreturn(::Type{T}, x::T) where {T} = x -cpyreturn(::Type{CPyPtr}, x::PyObject) = CPyPtr(pyptr(pyincref!(x))) -cpyreturn(::Type{T}, x::Number) where {T<:Number} = convert(T, x) -cpyreturn(::Type{T}, x::Ptr) where {T<:Ptr} = T(x) +""" + pyjlraw([T=PyObject,] x) -cpyerrval(::Type{Nothing}) = nothing -cpyerrval(::Type{T}) where {T<:Number} = zero(T)-one(T) -cpyerrval(::Type{T}) where {T<:Ptr} = T(C_NULL) - -### jlexception - -const pyjlexception = pylazyobject() do - attrs = pydict() - attrs["__module__"] = "julia" - attrs["__str__"] = pymethod(o -> begin - val = o.args[0] - pyisnone(val) && return "Unknown" - err, bt = pyjlgetvalue(val) - return sprint(showerror, err) - end) - t = pytypetype("Exception", (pyexception,), attrs) - t -end -export pyjlexception - -### jlfunction - -@kwdef struct CPyJlFunctionObject - ob_base :: C.PyObject = C.PyObject() - call :: Ptr{Cvoid} = C_NULL -end - -cpyjlfunction_call(o::CPyPtr, args::CPyPtr, kwargs::CPyPtr) = ccall(UnsafePtr{CPyJlFunctionObject}(o).call[!], CPyPtr, (CPyPtr, CPyPtr), args, kwargs) -function cpyjlfunction_dealloc(o::CPyPtr) - delete!(PYJLGCCACHE, o) - ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (CPyPtr,), o) - nothing -end - -const pyjlfunctiontype = pylazyobject() do - # make the type - c = [] - t = cpynewtype!(c; - name = "julia.Function", - base = pyobjecttype, - basicsize = sizeof(CPyJlFunctionObject), - flags = C.Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? C.Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), - call = @cfunction(cpyjlfunction_call, CPyPtr, (CPyPtr, CPyPtr, CPyPtr)), - dealloc = @cfunction(cpyjlfunction_dealloc, Cvoid, (CPyPtr,)), - ) - # put into a 0-dim array and take a pointer - ta = fill(t) - ptr = pointer(ta) - # ready the type - check(C.PyType_Ready(ptr)) - # success - PYJLGCCACHE[CPyPtr(ptr)] = push!(c, ta) - pyborrowedobject(ptr) -end -export pyjlfunctiontype - -struct cpyjlfunction_call_impl{F} - f :: F -end -(f::cpyjlfunction_call_impl)(_args::CPyPtr, _kwargs::CPyPtr) = cpycatch() do - if _kwargs != C_NULL - args = pyborrowedobject(_args) - kwargs = Dict{Symbol,PyObject}(Symbol(string(k)) => v for (k,v) in pyborrowedobject(_kwargs).items()) - pyobject(f.f(args...; kwargs...)) - elseif _args != C_NULL - args = pyborrowedobject(_args) - nargs = length(args) - if nargs == 0 - pyobject(f.f()) - elseif nargs == 1 - pyobject(f.f(args[0])) - elseif nargs == 2 - pyobject(f.f(args[0], args[1])) - elseif nargs == 3 - pyobject(f.f(args[0], args[1], args[2])) - else - pyobject(f.f(args...)) - end - else - pyobject(f.f()) - end -end - -function pyjlfunction(f) - # allocate an object - o = check(C._PyObject_New(pyjlfunctiontype)) - # set value - p = UnsafePtr{CPyJlFunctionObject}(pyptr(o)) - c = [] - p.call[] = cacheptr!(c, @cfunction($(cpyjlfunction_call_impl(f)), CPyPtr, (CPyPtr, CPyPtr))) - PYJLGCCACHE[pyptr(o)] = c - # done - return o -end -export pyjlfunction - -### jlbasevalue - -@kwdef struct CPyJlValueObject{T} - ob_base :: C.PyObject = C.PyObject() - value :: Ptr{Cvoid} = C_NULL - weaklist :: CPyPtr = C_NULL -end -const CPyJlPtr{T} = Ptr{CPyJlValueObject{T}} - -function cpyjlvalue_dealloc(o::CPyPtr) - delete!(PYJLGCCACHE, o) - UnsafePtr{CPyJlValueObject{Any}}(o).weaklist[!] == C_NULL || C.PyObject_ClearWeakRefs(o) - ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (CPyPtr,), o) - nothing -end - -function cpyjlvalue_new(t::CPyPtr, args::CPyPtr, kwargs::CPyPtr) - o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], CPyPtr, (CPyPtr, C.Py_ssize_t), t, 0) - if o != C_NULL - p = UnsafePtr{CPyJlValueObject{Any}}(o) - p.value[] = C_NULL - p.weaklist[] = C_NULL - end - o -end - -cpyjlvalue_get_buffer(_o::CPyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) = cpycatch(Cint) do - o = pyborrowedobject(_o) - t = pytype(o) - if pyhasattr(t, "__jl_enable_buffer__") && pytruth(t.__jl_enable_buffer__) - pyjl_get_buffer(o, buf, flags) - else - pythrow(pybuffererror("Buffer protocol not supported by '$(t.__name__)'")) - end -end - -function cpyjlvalue_release_buffer(_o::CPyPtr, buf::Ptr{C.Py_buffer}) - delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[]) - nothing -end - -const pyjlbasetype = pylazyobject() do - # make the type - c = [] - t = cpynewtype!(c; - name = "julia.ValueBase", - basicsize = sizeof(CPyJlValueObject{Any}), - new = @cfunction(cpyjlvalue_new, CPyPtr, (CPyPtr, CPyPtr, CPyPtr)), - dealloc = @cfunction(cpyjlvalue_dealloc, Cvoid, (CPyPtr,)), - flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? C.Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), - weaklistoffset = fieldoffset(CPyJlValueObject{Any}, 3), - getattro = C.pyglobal(:PyObject_GenericGetAttr), - setattro = C.pyglobal(:PyObject_GenericSetAttr), - doc = "A Julia value with no semantics.", - as_buffer = (get=@cfunction(cpyjlvalue_get_buffer, Cint, (CPyPtr, Ptr{C.Py_buffer}, Cint)), release=@cfunction(cpyjlvalue_release_buffer, Cvoid, (CPyPtr, Ptr{C.Py_buffer}))), - ) - # put into a 0-dim array and take a pointer - ta = fill(t) - ptr = pointer(ta) - # ready the type - check(C.PyType_Ready(ptr)) - # success - PYJLGCCACHE[CPyPtr(ptr)] = push!(c, ta) - pyborrowedobject(ptr) -end -export pyjlbasetype - -function pyjlnewvalue(x, t::PyObject) - pyissubclass(t, pyjlbasetype) || error("Expecting a subtype of 'julia.ValueBase'") - # allocate - o = t() - # set value - p = UnsafePtr{CPyJlValueObject{Any}}(pyptr(o)) - p.value[], PYJLGCCACHE[pyptr(o)] = pointer_from_obj(x) - # done - return o -end - -pyjlbase(x) = pyjlnewvalue(x, pyjlbasetype) -export pyjlbase - -pyisjl(o::PyObject) = pyisinstance(o, pyjlbasetype) -export pyisjl - -cpyjlgetvalue(o::CPyJlPtr{T}) where {T} = Base.unsafe_pointer_to_objref(UnsafePtr(o).value[!]) :: T -cpyjlgetvalue(o::Ptr) = cpyjlgetvalue(CPyJlPtr{Any}(o)) - -function pyjlgetvalue(o::PyObject, ::Type{T}=Any) where {T} - pyisinstance(o, pyjlbasetype) || pythrow(pytypeerror("Expecting a Python 'julia.ValueBase'")) - ptr = UnsafePtr(CPyJlPtr{Any}(pyptr(o))).value[!] - ptr == C_NULL && pythrow(pyvalueerror("Value is NULL")) - Base.unsafe_pointer_to_objref(ptr)::T -end -export pyjlgetvalue - -### jlrawvalue - -const PYJLRAWTYPES = Dict{Type,PyObject}() - -pyjlrawtype(::Type{T}) where {T} = get!(PYJLRAWTYPES, T) do - base = pyjl_supertype(T)===nothing ? pyjlbasetype : pyjlrawtype(pyjl_supertype(T)) - attrs = pydict() - attrs["__slots__"] = pytuple() - attrs["__module__"] = "julia" - attrs["__repr__"] = pymethod(o -> "") - attrs["__str__"] = pymethod(o -> string(pyjlgetvalue(o, T))) - attrs["__doc__"] = """ - A Julia '$T' with basic Julia semantics. - """ - # Logic - # Note that comparisons use isequal and isless, so that hashing is supported - attrs["__eq__"] = pymethod((o, x) -> pybool(isequal(pyjlgetvalue(o, T), pyconvert(Any, x)))) - attrs["__lt__"] = pymethod((o, x) -> pybool(isless(pyjlgetvalue(o, T), pyconvert(Any, x)))) - attrs["__bool__"] = pymethod(o -> (v=pyjlgetvalue(o, T); v isa Bool ? pybool(v) : pythrow(pytypeerror("Only 'Bool' can be tested for truthyness")))) - attrs["__hash__"] = pymethod(o -> pyint(hash(pyjlgetvalue(o, T)))) - # Containers - attrs["__contains__"] = pymethod((o, v) -> pybool(pyconvert(Any, v) in pyjlgetvalue(o, T))) - attrs["__getitem__"] = pymethod((o, i) -> o.__jl_wrap_result(pyjlraw(getindex(pyjlgetvalue(o, T), (pyistuple(i) ? [pyconvert(Any, j) for j in i] : [pyconvert(Any, i)])...)))) - attrs["__setitem__"] = pymethod((o, i, v) -> (setindex!(pyjlgetvalue(o, T), pyconvert(Any, v), (pyistuple(i) ? [pyconvert(Any, j) for j in i] : [pyconvert(Any, i)])...); pynone)) - attrs["__iter__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(Iterator(pyjlgetvalue(o, T), nothing)))) - if T <: Iterator - attrs["__next__"] = pymethod(_o -> begin - o = pyjlgetvalue(_o, T) - if o.st === nothing - it = iterate(o.val) - else - it = iterate(o.val, something(o.st)) - end - if it === nothing - pythrow(pystopiteration()) - else - x, st = it - o.st = Some(st) - return pyjlraw(x) - end - end) - end - # Function call - # TODO: keywords - attrs["__call__"] = pymethod((o, args...; kwargs...) -> begin - f = pyjlgetvalue(o, T) - jargs = map(x->pyconvert(Any, x), args) - r = if isempty(kwargs) - f(jargs...) - else - jkwargs = Tuple(k => pyconvert(Any, v) for (k,v) in kwargs) - f(jargs...; jkwargs...) - end - o.__jl_wrap_result(pyjlraw(r)) - end) - # Attributes - attrs["__dir__"] = pymethod(o -> begin - d = pyobjecttype.__dir__(o) - d.extend(pylist([pyjl_attrname_jl2py(string(k)) for k in propertynames(pyjlgetvalue(o, T), false)])) - d - end) - attrs["__getattr__"] = pymethod((_o, _k) -> begin - # first do a generic lookup - _x = C.PyObject_GenericGetAttr(_o, _k) - if _x != C_NULL - return pynewobject(_x) - elseif !pyerroccurred(pyattributeerror) - pythrow() - end - st = pyerrfetch() - # try to get a julia property with this name - o = pyjlgetvalue(_o, T) - k = Symbol(pyjl_attrname_py2jl(pystr_asjuliastring(_k))) - try - return _o.__jl_wrap_result(pyjlraw(getproperty(o, k))) - catch err - if (err isa UndefVarError) || (err isa ErrorException && occursin("has no field", err.msg)) - throw(PythonRuntimeError(st...)) - else - rethrow() - end - end - end) - attrs["__setattr__"] = pymethod((_o, _k, _v) -> begin - # first do a generic lookup - _x = C.PyObject_GenericSetAttr(_o, _k, _v) - if _x != -1 - return pynone - elseif !pyerroccurred(pyattributeerror) - pythrow() - end - st = pyerrfetch() - # try to set a julia property with this name - o = pyjlgetvalue(_o, T) - k = Symbol(pyjl_attrname_py2jl(pystr_asjuliastring(_k))) - v = pyconvert(Any, _v) - try - setproperty!(o, k, v) - return pynone - catch err - if (err isa UndefVarError) || (err isa ErrorException && occursin("has no field", err.msg)) - throw(PythonRuntimeError(st...)) - else - rethrow() - end - end - end) - # Arithmetic - attrs["__add__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) + pyconvert(Any, x)))) - attrs["__sub__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) - pyconvert(Any, x)))) - attrs["__mul__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) * pyconvert(Any, x)))) - attrs["__truediv__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) / pyconvert(Any, x)))) - attrs["__floordiv__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(fld(pyjlgetvalue(o, T), pyconvert(Any, x))))) - attrs["__mod__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(mod(pyjlgetvalue(o, T), pyconvert(Any, x))))) - attrs["__pow__"] = pymethod((o, x, m=pynone) -> o.__jl_wrap_result(pyjlraw(pyisnone(m) ? pyjlgetvalue(o, T) ^ pyconvert(Any, x) : powermod(pyjlgetvalue(o, T), pyconvert(Any, x), pyjlgetvalue(m))))) - attrs["__lshift__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) << pyconvert(Any, x)))) - attrs["__rshift__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) >> pyconvert(Any, x)))) - attrs["__and__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) & pyconvert(Any, x)))) - attrs["__xor__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) ⊻ pyconvert(Any, x)))) - attrs["__or__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) | pyconvert(Any, x)))) - attrs["__neg__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(-pyjlgetvalue(o, T)))) - attrs["__pos__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(+pyjlgetvalue(o, T)))) - attrs["__abs__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(abs(pyjlgetvalue(o, T))))) - attrs["__invert__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(~pyjlgetvalue(o, T)))) - attrs["__complex__"] = pymethod(o -> pycomplex(convert(Complex{Cdouble}, pyjlgetvalue(o, T)))) - attrs["__float__"] = pymethod(o -> pyfloat(convert(Cdouble, pyjlgetvalue(o, T)))) - attrs["__int__"] = attrs["__index__"] = pymethod(o -> pyint(convert(Integer, pyjlgetvalue(o, T)))) - # Julia-specific - attrs["__jl_getfield"] = pymethod((o, k) -> o.__jl_wrap_result(pyjlraw(getfield(pyjlgetvalue(o, T), pyjlraw_propertyname(k))))) - attrs["__jl_getprop"] = pymethod((o, k) -> o.__jl_wrap_result(pyjlraw(getproperty(pyjlgetvalue(o, T), pyjlraw_propertyname(k))))) - attrs["__jl_setfield"] = pymethod((o, k, v) -> (setfield!(pyjlgetvalue(o, T), pyjlraw_propertyname(k), pyjlgetvalue(v)); pynone)) - attrs["__jl_setprop"] = pymethod((o, k, v) -> (setproperty!(pyjlgetvalue(o, T), pyjlraw_propertyname(k), pyjlgetvalue(v)); pynone)) - attrs["__jl_prop"] = pymethod((o, k, v=nothing) -> v===nothing ? o.__jl_getprop(k) : o.__jl_setprop(k, v)) - attrs["__jl_field"] = pymethod((o, k, v=nothing) -> v===nothing ? o.__jl_getfield(k) : o.__jl_setfield(k, v)) - attrs["__jl_typeof"] = isconcretetype(T) ? pymethod(o -> o.__jl_wrap_result(pyjlraw(T))) : pymethod(o -> o.__jl_wrap_result(pyjlraw(typeof(pyjlgetvalue(o, T))))) - attrs["__jl_wrap_result"] = pymethod((o, v) -> v.__jl_raw()) - attrs["__jl_pyobject"] = pymethod(o -> pyobject(pyjlgetvalue(o, T))) - attrs["__jl_raw"] = pymethod(o -> o) - attrs["__jl_curly"] = pymethod((o, args...) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T){map(x->pyconvert(Any, x), args)...}))) - # Done - pytypetype("RawValue[$T]", (base,), attrs) -end -export pyjlrawtype - -pyjlraw_propertyname(k::PyObject) = pyisstr(k) ? Symbol(pystr_asjuliastring(k)) : pyisint(k) ? pyint_tryconvert(Int, k) : pyconvert(Any, k) - -pyjlraw(x, ::Type{T}) where {T} = x isa T ? pyjlnewvalue(x, pyjlrawtype(T)) : error("expecting a `$T`") -pyjlraw(x) = pyjlraw(x, typeof(x)) +Wrap `x` as a Python `julia.RawValue` object. +""" +pyjlraw(::Type{T}, x) where {T} = checknullconvert(T, C.PyJuliaRawValue_New(x)) +pyjlraw(x) = pyjlraw(PyObject, x) export pyjlraw -### jlvalue +""" + pyjl([T=PyObject,] x) -const PYJLTYPES = Dict{Type,PyObject}() - -function pyjl_supertypes(::Type{T}) where {T} - r = [] - S = T - while S !== nothing - push!(r, S) - S = pyjl_supertype(S) - end - r -end - -pyjl_supertype(::Type{T}) where {T} = supertype(T) -pyjl_supertype(::Type{Any}) = nothing -pyjl_supertype(::Type{Type{T}}) where {T} = Any -pyjl_supertype(::Type{Type}) = Any -pyjl_supertype(::Type{DataType}) = Type -pyjl_supertype(::Type{UnionAll}) = Type -pyjl_supertype(::Type{Union}) = Type - -pyjl_valuetype(::Type{T}) where {T} = T - -abstract type PyJlSubclass{S} end - -pyjl_supertype(::Type{T}) where {S, T<:PyJlSubclass{S}} = - if supertype(T) <: PyJlSubclass - if supertype(supertype(T)) <: PyJlSubclass - supertype(T) - else - S - end - else - error("cannot instantiate `PyJlSubclass`") - end -pyjl_supertype(::Type{T}) where {T<:PyJlSubclass} = error() - -pyjl_valuetype(::Type{T}) where {S, T<:PyJlSubclass{S}} = pyjl_valuetype(S) -pyjl_valuetype(::Type{T}) where {T<:PyJlSubclass} = error() - -pyjltype(::Type{T}) where {T} = get!(PYJLTYPES, T) do - # Bases - Ss = pyjl_supertypes(T) - V = pyjl_valuetype(T) - bases = [pyjlrawtype(V)] - length(Ss) == 1 || pushfirst!(bases, pyjltype(Ss[2])) - mixin = pyjl_mixin(T) - pyisnone(mixin) || push!(bases, mixin) - - # Attributes - attrs = Dict{String,PyObject}() - attrs["__module__"] = "julia" - attrs["__slots__"] = pytuple() - attrs["__jl_wrap_result"] = pymethod((o, x) -> x.__jl_pyobject()) - attrs["__jl_raw"] = pymethod(o -> pyjlraw(pyjlgetvalue(o, V))) - for S in reverse(Ss) - V <: pyjl_valuetype(S) || error() - pyjl_addattrs(attrs, S, V) - end - - # Make the type - t = pytypetype("Value[$T]", pytuple_fromiter(bases), pydict_fromstringiter(attrs)) - - # Register an abstract base class - abc = pyjl_abc(T) - pyisnone(abc) || abc.register(t) - - # Done - return t -end -export pyjltype - -pyjl(x, ::Type{T}) where {T} = x isa pyjl_valuetype(T) ? pyjlnewvalue(x, pyjltype(T)) : error("expecting a `$(pyjl_valuetype(T))`") -pyjl(x) = pyjl(x, typeof(x)) +Wrap `x` as a Python `julia.AnyValue` (or subclass) object. +""" +pyjl(::Type{T}, x) where {T} = checknullconvert(T, C.PyJuliaValue_From(x)) +pyjl(x) = pyjl(PyObject, x) export pyjl -### Any - -pyjl_abc(::Type) = pynone -pyjl_mixin(::Type) = pynone - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Any, V<:T} - t["__repr__"] = pymethod(o -> "") - t["__str__"] = pymethod(o -> string(pyjlgetvalue(o, V))) - t["__doc__"] = """ - A Julia '$T' with Python semantics. - """ - if hasmethod(length, Tuple{V}) - t["__len__"] = pymethod(o -> length(pyjlgetvalue(o, V))) - end - if hasmethod(in, Tuple{Union{}, V}) - t["__contains__"] = pymethod((_o, _x) -> begin - o = pyjlgetvalue(_o, V) - x = pytryconvert_element(o, _x) - x === PyConvertFail() ? false : in(x, o) - end) - end - if hasmethod(reverse, Tuple{V}) - t["__reversed__"] = pymethod(o -> pyjl(reverse(pyjlgetvalue(o, V)))) - end - if hasmethod(iterate, Tuple{V}) - t["__iter__"] = pymethod(o -> pyjl(Iterator(pyjlgetvalue(o, V), nothing))) - end - if hasmethod(getindex, Tuple{V, Union{}}) - t["__getitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert_indices(o, _k) - if k === PyConvertFail() - pythrow(pytypeerror("invalid index")) - end - pyobject(o[k...]) - end) - end - if hasmethod(setindex!, Tuple{V, Union{}, Union{}}) - t["__setitem__"] = pymethod((_o, _k, _v) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert_indices(o, _k) - k === PyConvertFail && pythrow(pytypeerror("invalid index")) - v = pytryconvert_value(o, _v, k...) - v === PyConvertFail && pythrow(pytypeerror("invalid value")) - o[k...] = v - pynone - end) - end - # Comparisons - t["__eq__"] = pymethod((o,x) -> pyjlgetvalue(o, V) == pyconvert(Any, x)) - t["__ne__"] = pymethod((o,x) -> pyjlgetvalue(o, V) != pyconvert(Any, x)) - t["__le__"] = pymethod((o,x) -> pyjlgetvalue(o, V) <= pyconvert(Any, x)) - t["__lt__"] = pymethod((o,x) -> pyjlgetvalue(o, V) < pyconvert(Any, x)) - t["__ge__"] = pymethod((o,x) -> pyjlgetvalue(o, V) >= pyconvert(Any, x)) - t["__gt__"] = pymethod((o,x) -> pyjlgetvalue(o, V) > pyconvert(Any, x)) - # MIME display - for (mime, method) in ( - (MIME"text/html"(), "_repr_html_"), - (MIME"text/markdown"(), "_repr_markdown_"), - (MIME"text/json"(), "_repr_json_"), - (MIME"application/javascript"(), "_repr_javascript_"), - (MIME"application/pdf"(), "_repr_pdf_"), - (MIME"image/jpeg"(), "_repr_jpeg_"), - (MIME"image/png"(), "_repr_png_"), - (MIME"image/svg+xml"(), "_repr_svg_"), - (MIME"text/latex"(), "_repr_latex_")) - - t[method] = pymethod(_o -> begin - o = pyjlgetvalue(_o, V) - if showable(mime, o) - io = IOBuffer() - show(io, mime, o) - r = pybytes(take!(io)) - istextmime(mime) ? r.decode("utf8") : r - else - pynone - end - end) - end -end - -pyjl_attrname_py2jl(x::AbstractString) = - replace(x, r"_[b]+$" => s -> "!"^(length(s)-1)) - -pyjl_attrname_jl2py(x::AbstractString) = - replace(x, r"!+$" => s -> "_" * "b"^(length(s))) - -### Nothing & Missing (falsy) - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Nothing,Missing}, V<:T} - t["__bool__"] = pymethod(o -> pyfalse) -end - -### Module - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Module, V<:T} - t["__dir__"] = pymethod(_o -> begin - d = pyjltype(Any).__dir__(_o) - o = pyjlgetvalue(_o, V) - for n in names(o, all=true, imported=true) - d.append(pyjl_attrname_jl2py(string(n))) - end - for m in ccall(:jl_module_usings, Any, (Any,), o)::Vector - for n in names(m) - d.append(pyjl_attrname_jl2py(string(n))) - end - end - return d - end) -end - -### Type - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Type, V<:T} - t["curly"] = pymethod((o, args...) -> o.__jl_curly(args...)) -end - -### Iterator (as Iterator) - -mutable struct Iterator{T} - val :: T - st :: Union{Nothing, Some} -end - -pyjl_mixin(::Type{T}) where {T<:Iterator} = pyiteratorabc - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Iterator, V<:T} - t["__iter__"] = pymethod(o -> o) - t["__next__"] = pymethod(_o -> begin - o = pyjlgetvalue(_o, V) - if o.st === nothing - it = iterate(o.val) - else - it = iterate(o.val, something(o.st)) - end - if it === nothing - pythrow(pystopiteration()) - else - x, st = it - o.st = Some(st) - return pyobject(x) - end - end) -end - -### Number - -pyjl_mixin(::Type{T}) where {T<:Number} = pynumbersmodule.Number -pyjl_mixin(::Type{T}) where {T<:Complex} = pynumbersmodule.Complex -pyjl_mixin(::Type{T}) where {T<:Real} = pynumbersmodule.Real -pyjl_mixin(::Type{T}) where {T<:Rational} = pynumbersmodule.Rational -pyjl_mixin(::Type{T}) where {T<:Integer} = pynumbersmodule.Integral - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Number, V<:T} - t["__bool__"] = pymethod(o -> pybool(!iszero(pyjlgetvalue(o, V)))) - t["__pos__"] = pymethod(o -> pyobject(+pyjlgetvalue(o, V))) - t["__neg__"] = pymethod(o -> pyobject(-pyjlgetvalue(o, V))) -end - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Complex,Real}, V<:T} - t["real"] = pyproperty(o -> pyobject(real(pyjlgetvalue(o, V)))) - t["imag"] = pyproperty(o -> pyobject(imag(pyjlgetvalue(o, V)))) - t["conjugate"] = pymethod(o -> pyobject(conj(pyjlgetvalue(o, V)))) - t["__abs__"] = pymethod(o -> pyobject(abs(pyjlgetvalue(o, V)))) - t["__complex__"] = pymethod(o -> pycomplex(pyjlgetvalue(o, V))) - if V<:Real - t["__float__"] = pymethod(o -> pyfloat(pyjlgetvalue(o, V))) - t["__trunc__"] = pymethod(o -> pyint(trunc(BigInt, pyjlgetvalue(o, V)))) - t["__round__"] = pymethod((o,n=pynone) -> pyisnone(n) ? pyint(round(BigInt, pyjlgetvalue(o, V))) : pyjl(round(pyjlgetvalue(o, V), digits=pyconvert(Int, n)))) - t["__floor__"] = pymethod(o -> pyint(floor(BigInt, pyjlgetvalue(o, V)))) - t["__ceil__"] = pymethod(o -> pyint(ceil(BigInt, pyjlgetvalue(o, V)))) - end -end - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Integer,Rational}, V<:T} - t["numerator"] = pyproperty(o -> pyobject(numerator(pyjlgetvalue(o, V)))) - t["denominator"] = pyproperty(o -> pyobject(denominator(pyjlgetvalue(o, V)))) - if V<:Integer - t["__int__"] = pymethod(o -> pyint(pyjlgetvalue(o, V))) - t["__invert__"] = pymethod(o -> pyobject(~pyjlgetvalue(o, V))) - end -end - -### Dict (as Mapping) - -pyjl_mixin(::Type{T}) where {T<:AbstractDict} = pymutablemappingabc - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractDict, V<:T} - t["__iter__"] = pymethod(o -> pyjl(Iterator(keys(pyjlgetvalue(o, V)), nothing))) - t["__getitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert(keytype(o), _k) - if k === PyConvertFail() || !haskey(o, k) - pythrow(pykeyerror(_k)) - end - pyobject(o[k]) - end) - t["__setitem__"] = pymethod((_o, _k, _v) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert(keytype(o), _k) - if k === PyConvertFail() - pythrow(pykeyerror(_k)) - end - v = pytryconvert(valtype(o), _v) - v === PyConvertFail() && pythrow(pytypeerror("invalid value of type '$(pytype(_v).__name__)'")) - o[k] = v - pynone - end) - t["__delitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert(keytype(o), _k) - if k === PyConvertFail() || !haskey(o, k) - pythrow(pykeyerror(_k)) - end - delete!(o, k) - pynone - end) - t["__contains__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pytryconvert(keytype(o), _k) - k === PyConvertFail() ? false : haskey(o, k) - end) - t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) -end - -### Set (as Set) - -pyjl_mixin(::Type{T}) where {T<:AbstractSet} = pymutablesetabc - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractSet, V<:T} - t["add"] = pymethod((_o, _v) -> begin - o = pyjlgetvalue(_o, V) - v = pytryconvert(eltype(o), _v) - v === PyConvertFail() && pythrow(pytypeerror("Invalid value of type '$(pytype(v).__name__)'")) - push!(o, v) - pynone - end) - t["discard"] = pymethod((_o, _v) -> begin - o = pyjlgetvalue(_o, V) - v = pytryconvert(eltype(o), _v) - v === PyConvertFail() || delete!(o, v) - pynone - end) - t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) -end - -### Array (as Collection) -### Vector (as Sequence) - -pyjl_mixin(::Type{T}) where {T<:AbstractArray} = pycollectionabc -pyjl_mixin(::Type{T}) where {T<:AbstractVector} = pymutablesequenceabc - -function pyjl_axisidx(ax, k::PyObject) - # slice - if pyisslice(k) - error("slicing not implemented") - end - # convert to int - if pyisint(k) - # nothing to do - elseif pyhasattr(k, "__index__") - k = k.__index__() - else - pythrow(pytypeerror("Indices must be 'int' or 'slice', not '$(pytype(k).__name__)'")) - end - # convert to julia int - i = check(C.PyLong_AsLongLong(k), true) - # negative indexing - j = i<0 ? i+length(ax) : i - # bounds check - 0 ≤ j < length(ax) || pythrow(pyindexerror("Index out of range")) - # adjust for zero-up indexing - j + first(ax) -end - -function pyjl_arrayidxs(o::AbstractArray{T,N}, k::PyObject) where {T,N} - if pyistuple(k) - N == pylen(k) && return ntuple(i -> pyjl_axisidx(axes(o, i), k[i-1]), N) - else - N == 1 && return (pyjl_axisidx(axes(o, 1), k),) - end - pythrow(pytypeerror("Expecting a tuple of $N indices")) -end - -pyjl_isbufferabletype(::Type{T}) where {T} = - T in (Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}, Bool, Ptr{Cvoid}) -pyjl_isbufferabletype(::Type{T}) where {T<:Tuple} = - isconcretetype(T) && Base.allocatedinline(T) && all(pyjl_isbufferabletype, fieldtypes(T)) -pyjl_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = - pyjl_isbufferabletype(T) - -pyjl_isarrayabletype(::Type{T}) where {T} = - T in (UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64, Bool, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}) - -islittleendian() = Base.ENDIAN_BOM == 0x04030201 ? true : Base.ENDIAN_BOM == 0x01020304 ? false : error() - -function pytypestrformat(::Type{T}) where {T} - c = islittleendian() ? '<' : '>' - T == Int8 ? ("$(c)i1", pynone) : - T == UInt8 ? ("$(c)u1", pynone) : - T == Int16 ? ("$(c)i2", pynone) : - T == UInt16 ? ("$(c)u2", pynone) : - T == Int32 ? ("$(c)i4", pynone) : - T == UInt32 ? ("$(c)u4", pynone) : - T == Int64 ? ("$(c)i8", pynone) : - T == UInt64 ? ("$(c)u8", pynone) : - T == Float16 ? ("$(c)f2", pynone) : - T == Float32 ? ("$(c)f4", pynone) : - T == Float64 ? ("$(c)f8", pynone) : - error("not implemented") -end - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractArray, V<:T} - if hasmethod(pointer, Tuple{V}) && hasmethod(strides, Tuple{V}) - if pyjl_isbufferabletype(eltype(V)) - t["__jl_enable_buffer__"] = true - end - if pyjl_isarrayabletype(eltype(V)) - t["__array_interface__"] = pyproperty(_o -> begin - o = pyjlgetvalue(_o, V) - typestr, descr = pytypestrformat(eltype(o)) - pydict( - shape = size(o), - typestr = typestr, - descr = descr, - data = (UInt(pointer(o)), !ismutablearray(o)), - strides = strides(o) .* Base.aligned_sizeof(eltype(o)), - version = 3, - ) - end) - end - end - if V <: PyObjectArray - t["__array_interface__"] = pyproperty(_o -> begin - o = pyjlgetvalue(_o, V) - pydict( - shape = size(o), - typestr = "O", - data = (UInt(pointer(o.ptrs)), false), - strides = strides(o.ptrs) .* sizeof(CPyPtr), - version = 3, - ) - end) - end - t["ndim"] = pyproperty(o -> pyint(ndims(pyjlgetvalue(o, V)))) - t["shape"] = pyproperty(o -> pytuple(map(pyint, size(pyjlgetvalue(o, V))))) - t["__array__"] = pymethod(o -> pyhasattr(o, "__array_interface__") ? pynumpy.asarray(o) : pynumpy.asarray(PyObjectArray(pyjlgetvalue(o, V)))) - t["__getitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pyjl_arrayidxs(o, _k) - pyobject(o[k...]) - end) - t["__setitem__"] = pymethod((_o, _k, _v) -> begin - o = pyjlgetvalue(_o, V) - k = pyjl_arrayidxs(o, _k) - v = pytryconvert(eltype(o), _v) - v === PyConvertFail() && pythrow(pytypeerror("Cannot assign value of type '$(pytype(_v).__name__)'")) - o[k...] = v - pynone - end) - t["copy"] = pymethod(o -> pyjl(copy(pyjlgetvalue(o, V)))) - if V <: AbstractVector - t["__delitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - k = pyjl_arrayidxs(o, _k) - deleteat!(o, k...) - pynone - end) - t["insert"] = pymethod((_o, _k, _v) -> begin - o = pyjlgetvalue(_o, V) - ax = axes(o, 1) - k = pyjl_axisidx(first(ax):(last(ax)+1), _k) - v = pytryconvert(eltype(o), _v) - v === PyConvertFail() && pythrow(pytypeerror("Cannot assign value of type '$(pytype(_v).__name__)'")) - insert!(o, k, v) - pynone - end) - t["sort"] = pymethod((_o; reverse=pyfalse, key=pynone) -> begin - o = pyjlgetvalue(_o) - rev = pytruth(reverse) - by = pyisnone(key) ? identity : key - sort!(o, rev=rev, by=by) - pynone - end) - t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) - end -end - -### Tuple & NamedTuple (as Sequence) - -pyjl_mixin(::Type{T}) where {T<:Union{Tuple,NamedTuple}} = pysequenceabc - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Tuple,NamedTuple}, V<:T} - t["__getitem__"] = pymethod((_o, _k) -> begin - o = pyjlgetvalue(_o, V) - if o isa NamedTuple && pyisstr(_k) - k = Symbol(string(_k)) - haskey(o, k) || pythrow(pykeyerror(_k)) - else - k = pyjl_axisidx(1:length(o), _k) - end - pyobject(o[k]) - end) -end - -### IO (as IOBase) - -pyjl_abc(::Type{T}) where {T<:IO} = pyiomodule.IOBase - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:IO, V} - t["close"] = if hasmethod(close, Tuple{V}) - pymethod(o -> (close(pyjlgetvalue(o, V)); pynone)) - else - pymethod(o -> pythrow(pyiounsupportedoperation("close"))) - end - t["closed"] = if hasmethod(isopen, Tuple{V}) - pyproperty(o -> pybool(!isopen(pyjlgetvalue(o, V)))) - else - pyproperty(o -> pythrow(pyiounsupportedoperation("closed"))) - end - t["fileno"] = if hasmethod(fd, Tuple{V}) - pymethod(o -> pyint(fd(pyjlgetvalue(o, V)))) - else - pymethod(o -> pythrow(pyiounsupportedoperation("fileno"))) - end - t["flush"] = pymethod(o -> (flush(pyjlgetvalue(o, V)); pynone)) - t["isatty"] = V<:Base.TTY ? pymethod(o -> pybool(true)) : pymethod(o -> pybool(false)) - t["readable"] = if hasmethod(isreadable, Tuple{V}) - pymethod(o -> pybool(isreadable(pyjlgetvalue(o, V)))) - else - pymethod(o -> pythrow(pyiounsupportedoperation("readable"))) - end - t["writable"] = if hasmethod(iswritable, Tuple{V}) - pymethod(o -> pybool(iswritable(pyjlgetvalue(o, V)))) - else - pymethod(o -> pythrow(pyiounsupportedoperation("writable"))) - end - t["tell"] = if hasmethod(position, Tuple{V}) - pymethod(o -> pyint(position(pyjlgetvalue(o, V)))) - else - pymethod(o -> pythrow(pyiounsupportedoperation("tell"))) - end - t["writelines"] = pymethod((o, lines) -> begin - wr = o.write - for line in lines - wr(line) - end - pynone - end) - t["seekable"] = if hasmethod(position, Tuple{V}) && hasmethod(seek, Tuple{V,Int}) && hasmethod(truncate, Tuple{V,Int}) - pymethod(o -> pybool(true)) - else - pymethod(o -> pybool(false)) - end - t["truncate"] = if hasmethod(truncate, Tuple{V, Int}) - pymethod((_o, _n=pynone) -> begin - o = pyjlgetvalue(_o, V) - n = pyisnone(_n) ? position(o) : pyconvert(Int, _n) - truncate(o, n) - pyint(n) - end) - else - pymethod((_o, _n=pynone) -> pythrow(pyiounsupportedoperation("truncate"))) - end - t["seek"] = if hasmethod(seek, Tuple{V, Int}) && hasmethod(position, Tuple{V}) - pymethod((_o, offset, whence=pyint(0)) -> begin - o = pyjlgetvalue(_o, V) - n = pyconvert(Int, offset) - w = pyconvert(Int, whence) - if w == 0 - seek(o, n) - elseif w == 1 - seek(o, position(o)+n) - elseif w == 2 - seekend(o) - seek(o, position(o)+n) - else - pythrow(pyvalueerror("Unsupported whence: $w")) - end - pyint(position(o)) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("seek"))) - end - t["__iter__"] = pymethod(o -> o) - t["__next__"] = pymethod(o -> (x=o.readline(); pylen(x)==0 ? pythrow(pystopiteration()) : x)) -end - -### RawIO (as RawIOBase) - -abstract type RawIO{V<:IO} <: PyJlSubclass{V} end - -pyjlrawio(o::IO) = pyjl(o, RawIO{typeof(o)}) - -pyjl_abc(::Type{T}) where {T<:RawIO} = pyiomodule.RawIOBase - -### BufferedIO (as BufferedIOBase) - -abstract type BufferedIO{V<:IO} <: PyJlSubclass{V} end - -pyjl_abc(::Type{T}) where {T<:BufferedIO} = pyiomodule.BufferedIOBase - -pyjlbufferedio(o::IO) = pyjl(o, BufferedIO{typeof(o)}) - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:BufferedIO, V} - t["detach"] = pymethod(o -> pythrow(pyiounsupportedoperation("detach"))) - t["readinto"] = if hasmethod(readbytes!, Tuple{V, Vector{UInt8}}) - pymethod((_o, _b) -> begin - b = PyBuffer(_b, C.PyBUF_WRITABLE) - o = pyjlgetvalue(_o, V) - n = readbytes!(o, unsafe_wrap(Array{UInt8}, Ptr{UInt8}(b.buf), b.len)) - pyint(n) - end) - else - pymethod((o, b) -> pythrow(pyiounsupportedoperation("readinto"))) - end - t["read"] = if hasmethod(read, Tuple{V}) && hasmethod(read, Tuple{V, Int}) - pymethod((_o, size=pynone) -> begin - n = pyconvert(Union{Int,Nothing}, size) - o = pyjlgetvalue(_o, V) - pybytes(convert(Vector{UInt8}, (n===nothing || n < 0) ? read(o) : read(o, n))) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("read"))) - end - t["write"] = if hasmethod(write, Tuple{V, Vector{UInt8}}) - pymethod((_o, _b) -> begin - b = PyBuffer(_b, C.PyBUF_SIMPLE) - o = pyjlgetvalue(_o, V) - n = write(o, unsafe_wrap(Array{UInt8}, Ptr{UInt8}(b.buf), b.len)) - pyint(n) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("write"))) - end - t["readline"] = if hasmethod(read, Tuple{V, Type{UInt8}}) - pymethod((_o, size=pynone) -> begin - n = pyconvert(Union{Int,Nothing}, size) - o = pyjlgetvalue(_o, V) - data = UInt8[] - while !eof(o) && (n===nothing || n < 0 || length(data) ≤ n) - c = read(o, UInt8) - push!(data, c) - c == 0x0A && break - end - pybytes(data) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("readline"))) - end -end - -### TextIO (as TextIOBase) - -abstract type TextIO{V<:IO} <: PyJlSubclass{V} end - -pyjl_abc(::Type{T}) where {T<:TextIO} = pyiomodule.TextIOBase - -pyjltextio(o::IO) = pyjl(o, TextIO{typeof(o)}) - -function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:TextIO, V} - t["encoding"] = pyproperty(o -> pystr("utf-8")) - t["errors"] = pyproperty(o -> pystr("strict")) - t["newlines"] = pyproperty(o -> pynone) - t["detach"] = pymethod(o -> pythrow(pyiounsupportedoperation("detach"))) - t["read"] = if hasmethod(read, Tuple{V, Type{Char}}) - pymethod((_o, size=pynone) -> begin - n = pytryconvert(Union{Nothing,Int}, size) - n === PyConvertFail() && pythrow(pytypeerror("'size' must be 'int' or 'None'")) - o = pyjlgetvalue(_o, V) - b = IOBuffer() - i = 0 - while !eof(o) && (n===nothing || n < 0 || i < n) - i += 1 - write(b, read(o, Char)) - end - seekstart(b) - pystr(read(b, String)) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("read"))) - end - t["readline"] = if hasmethod(read, Tuple{V, Type{Char}}) - pymethod((_o, size=pynone) -> begin - n = pytryconvert(Union{Nothing,Int}, size) - n === PyConvertFail() && pythrow(pytypeerror("'size' must be 'int' or 'None'")) - o = pyjlgetvalue(_o, V) - b = IOBuffer() - i = 0 - while !eof(o) && (n === nothing || n < 0 || i < n) - i += 1 - c = read(o, Char) - if c == '\n' - write(b, '\n') - break - elseif c == '\r' - write(b, '\n') - !eof(o) && peek(o, Char)=='\n' && read(o, Char) - break - else - write(b, c) - end - end - seekstart(b) - pystr(read(b, String)) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("readline"))) - end - t["write"] = if hasmethod(write, Tuple{V, String}) - pymethod((_o, _x) -> begin - x = pystr_asjuliastring(_x) - o = pyjlgetvalue(_o) - n = 0 - linesep = pystr_asjuliastring(pyosmodule.linesep) - i = firstindex(x) - while true - j = findnext('\n', x, i) - if j === nothing - y = SubString(x, i, lastindex(x)) - write(o, y) - n += length(y) - break - else - y = SubString(x, i, prevind(x, j)) - write(o, y) - write(o, linesep) - n += length(y) + 1 - i = nextind(x, j) - end - end - pyint(n) - end) - else - pymethod((args...) -> pythrow(pyiounsupportedoperation("write"))) - end -end - -### Buffer Protocol - -isflagset(flags, mask) = (flags & mask) == mask - -const PYJLBUFCACHE = Dict{Ptr{Cvoid}, Any}() - -function pyjl_get_buffer_impl(o, buf, flags, ptr, elsz, len, ndim, fmt, sz, strds, mutable) - b = UnsafePtr(buf) - c = [] - - # not influenced by flags: obj, buf, len, itemsize, ndim - b.obj[] = C_NULL - b.buf[] = ptr - b.itemsize[] = elsz - b.len[] = elsz * len - b.ndim[] = ndim - - # readonly - if isflagset(flags, C.PyBUF_WRITABLE) - if mutable - b.readonly[] = 1 - else - pythrow(pybuffererror("not writable")) - end - else - b.readonly[] = mutable ? 0 : 1 - end - - # format - if isflagset(flags, C.PyBUF_FORMAT) - b.format[] = cacheptr!(c, fmt) - else - b.format[] = C_NULL - end - - # shape - if isflagset(flags, C.PyBUF_ND) - b.shape[] = cacheptr!(c, C.Py_ssize_t[sz...]) - else - b.shape[] = C_NULL - end - - # strides - if isflagset(flags, C.PyBUF_STRIDES) - b.strides[] = cacheptr!(c, C.Py_ssize_t[(strds .* elsz)...]) - else - if size_to_cstrides(1, sz...) != strds - pythrow(pybuffererror("not C contiguous and strides not requested")) - end - b.strides[] = C_NULL - end - - # check contiguity - if isflagset(flags, C.PyBUF_C_CONTIGUOUS) - if size_to_cstrides(1, sz...) != strds - pythrow(pybuffererror("not C contiguous")) - end - end - if isflagset(flags, C.PyBUF_F_CONTIGUOUS) - if size_to_fstrides(1, sz...) != strds - pythrow(pybuffererror("not Fortran contiguous")) - end - end - if isflagset(flags, C.PyBUF_ANY_CONTIGUOUS) - if size_to_cstrides(1, sz...) != strds && size_to_fstrides(1, sz...) != strds - pythrow(pybuffererror("not contiguous")) - end - end - - # suboffsets - b.suboffsets[] = C_NULL - - # internal - cptr = Base.pointer_from_objref(c) - PYJLBUFCACHE[cptr] = c - b.internal[] = cptr +""" + pyjlgetvalue() +""" +pyjlgetvalue(o) = pyisjl(o) ? cpyop(C.PyJuliaValue_GetValue, o) : error("Not a Julia value") +export pyjlgetvalue - # obj - b.obj[] = pyptr(pyincref!(o)) - Cint(0) -end +""" + pyisjl(o) -pyjl_get_buffer(o, buf, flags) = pyjl_get_buffer(o, buf, flags, pyjlgetvalue(o)) -pyjl_get_buffer(o, buf, flags, x::AbstractArray) = - pyjl_get_buffer_impl(o, buf, flags, pointer(x), Base.aligned_sizeof(eltype(x)), length(x), ndims(x), pybufferformat(eltype(x)), size(x), strides(x), ismutablearray(x)) -pyjl_get_buffer(o, buf, flags, x::PyObjectArray) = - pyjl_get_buffer_impl(o, buf, flags, pointer(x.ptrs), sizeof(CPyPtr), length(x), ndims(x), "O", size(x), strides(x.ptrs), true) +True if `o` is a `julia.ValueBase` object. +""" +pyisjl(o) = cpyop(C.PyJuliaValue_Check, o) +export pyisjl diff --git a/src/list.jl b/src/list.jl deleted file mode 100644 index affaa490..00000000 --- a/src/list.jl +++ /dev/null @@ -1,25 +0,0 @@ -const pylisttype = pylazyobject(() -> pybuiltins.list) -export pylisttype - -pyislist(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_LIST_SUBCLASS) -export pyislist - -pylist() = check(C.PyList_New(0)) -pylist(args...; opts...) = pylisttype(args...; opts...) -pylist(x::Union{Tuple,AbstractVector}) = pylist_fromiter(x) -export pylist - -function pylist_fromiter(xs) - r = pylist() - for x in xs - xo = pyobject(x) - check(C.PyList_Append(r, xo)) - end - return r -end - -pycollist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pycollist(y) for y in eachslice(x; dims=N)) -export pycollist - -pyrowlist(x::AbstractArray{T,N}) where {T,N} = N==0 ? pyobject(x[]) : pylist_fromiter(pyrowlist(y) for y in eachslice(x; dims=1)) -export pyrowlist diff --git a/src/matplotlib.jl b/src/matplotlib.jl index 80008a80..31d184ec 100644 --- a/src/matplotlib.jl +++ b/src/matplotlib.jl @@ -1,6 +1,3 @@ -pymatplotlib = pylazyobject(() -> pyimport("matplotlib")) -pyplot = pylazyobject(() -> pyimport("matplotlib.pyplot")) - """ pyplotshow([fig]; close=true, [format]) @@ -10,11 +7,24 @@ If `close` is true, the figure is also closed. The `format` specifies the file format of the generated image. By default this is `pyplot.rcParams["savefig.format"]`. """ -function pyplotshow(fig; close::Bool=true, format::String=pyplot.rcParams.get("savefig.format", "png").jl!s) - fig = pyisinstance(fig, pyplot.Figure) ? PyObject(fig) : pyplot.figure(fig) - io = IOBuffer() - fig.savefig(io, format=format) - data = take!(io) +function pyplotshow(fig; close::Bool=true, format::String="") + @py ``` + import matplotlib.pyplot as plt, io + fig = $fig + if not isinstance(fig, plt.Figure): + fig = plt.figure(fig) + buf = io.BytesIO() + format = $format + if not format: + format = plt.rcParams.get("savefig.format", "png") + if format not in ["png", "jpg", "jpeg", "tif", "tiff", "svg", "pdf"]: + raise ValueError("Unsupported format: {}".format(format)) + fig.savefig(buf, format=format) + $(data::Vector{UInt8}) = buf.getvalue() + if $close: + plt.close(fig) + $(format::String) = format + ``` if format == "png" display(MIME"image/png"(), data) elseif format in ("jpg", "jpeg") @@ -26,13 +36,17 @@ function pyplotshow(fig; close::Bool=true, format::String=pyplot.rcParams.get("s elseif format == "pdf" display(MIME"application/pdf"(), data) else - error("Unsupported format: $(repr(format)) (try one of: png, jpg, jpeg, tif, tiff, svg, xml)") + error() end - close && pyplot.close(fig) nothing end function pyplotshow(; opts...) - for fig in pyplot.get_fignums() + @py ``` + import sys + plt = sys.modules.get("matplotlib.pyplot", None) + $(fignums::Vector{Int}) = [] if plt is None else plt.get_fignums() + ``` + for fig in fignums pyplotshow(fig; opts...) end end diff --git a/src/newtype.jl b/src/newtype.jl deleted file mode 100644 index cee95f32..00000000 --- a/src/newtype.jl +++ /dev/null @@ -1,79 +0,0 @@ -### CACHE - -cacheptr!(c, x::Ptr) = x -cacheptr!(c, x::String) = (push!(c, x); pointer(x)) -cacheptr!(c, x::AbstractString) = cacheptr!(c, String(x)) -cacheptr!(c, x::Array) = (push!(c, x); pointer(x)) -cacheptr!(c, x::AbstractArray) = cacheptr!(c, Array(x)) -cacheptr!(c, x::PyObject) = (push!(c, x); pyptr(x)) -cacheptr!(c, x::Base.CFunction) = (push!(c, x); Base.unsafe_convert(Ptr{Cvoid}, x)) - -### PROTOCOLS - -cpynewnumbermethods!(c, x::C.PyNumberMethods) = x -cpynewnumbermethods!(c; opts...) = C.PyNumberMethods(; opts...) -cpynewnumbermethods!(c, x::Dict) = cpynewnumbermethods!(c; x...) -cpynewnumbermethods!(c, x::NamedTuple) = cpynewnumbermethods!(c; x...) - -cpynewmappingmethods!(c, x::C.PyMappingMethods) = x -cpynewmappingmethods!(c; opts...) = C.PyMappingMethods(; opts...) -cpynewmappingmethods!(c, x::Dict) = cpynewmappingmethods!(c; x...) -cpynewmappingmethods!(c, x::NamedTuple) = cpynewmappingmethods!(c; x...) - -cpynewsequencemethods!(c, x::C.PySequenceMethods) = x -cpynewsequencemethods!(c; opts...) = C.PySequenceMethods(; opts...) -cpynewsequencemethods!(c, x::Dict) = cpynewsequencemethods!(c; x...) -cpynewsequencemethods!(c, x::NamedTuple) = cpynewsequencemethods!(c; x...) - -cpynewbufferprocs!(c, x::C.PyBufferProcs) = x -cpynewbufferprocs!(c; opts...) = C.PyBufferProcs(; opts...) -cpynewbufferprocs!(c, x::Dict) = cpynewbufferprocs!(c; x...) -cpynewbufferprocs!(c, x::NamedTuple) = cpynewbufferprocs!(c; x...) - -cpynewmethod!(c, x::C.PyMethodDef) = x -cpynewmethod!(c; name=C_NULL, meth=C_NULL, flags=0, doc=C_NULL) = - C.PyMethodDef( - name = name isa Ptr ? name : cacheptr!(c, string(name)), - meth = meth, - flags = flags, - doc = doc isa Ptr ? doc : cacheptr!(c, string(doc)), - ) -cpynewmethod!(c, x::Dict) = cpynewmethod!(c; x...) -cpynewmethod!(c, x::NamedTuple) = cpynewmethod!(c; x...) - -cpynewgetset!(c, x::C.PyGetSetDef) = x -cpynewgetset!(c; name=C_NULL, get=C_NULL, set=C_NULL, doc=C_NULL, closure=C_NULL) = - C.PyGetSetDef( - name = name isa Ptr ? name : cacheptr!(c, string(name)), - get = get, - set = set, - doc = doc isa Ptr ? doc : cacheptr!(c, string(doc)), - closure = closure, - ) -cpynewgetset!(c, x::Dict) = cpynewgetset!(c; x...) -cpynewgetset!(c, x::NamedTuple) = cpynewgetset!(c; x...) - -### NEW TYPE - -function cpynewtype!(c; type=C_NULL, name, as_number=C_NULL, as_mapping=C_NULL, as_sequence=C_NULL, as_buffer=C_NULL, methods=C_NULL, getset=C_NULL, opts...) - type = cacheptr!(c, type) - name = cacheptr!(c, name) - as_number = as_number isa Ptr ? as_number : cacheptr!(c, fill(cpynewnumbermethods!(c, as_number))) - as_mapping = as_mapping isa Ptr ? as_mapping : cacheptr!(c, fill(cpynewmappingmethods!(c, as_mapping))) - as_sequence = as_sequence isa Ptr ? as_sequence : cacheptr!(c, fill(cpynewsequencemethods!(c, as_sequence))) - as_buffer = as_buffer isa Ptr ? as_buffer : cacheptr!(c, fill(cpynewbufferprocs!(c, as_buffer))) - methods = - if methods isa Ptr - methods - else - cacheptr!(c, [[cpynewmethod!(c, m) for m in methods]; C.PyMethodDef()]) - end - getset = - if getset isa Ptr - getset - else - cacheptr!(c, [[cpynewgetset!(c, m) for m in getset]; C.PyGetSetDef()]) - end - newopts = Dict(k => (v isa Union{AbstractString,AbstractArray,PyObject,Base.CFunction} ? cacheptr!(c, v) : v) for (k,v) in pairs(opts)) - C.PyTypeObject(; ob_base=C.PyVarObject(ob_base=C.PyObject(type=type)), name=name, as_number=as_number, as_mapping=as_mapping, as_sequence=as_sequence, as_buffer=as_buffer, methods, getset, newopts...) -end diff --git a/src/none.jl b/src/none.jl deleted file mode 100644 index f5a70c38..00000000 --- a/src/none.jl +++ /dev/null @@ -1,12 +0,0 @@ -const pynone = pylazyobject(() -> pybuiltins.None) -export pynone - -const pynonetype = pylazyobject(() -> pytype(pynone)) -export pynonetype - -pyisnone(o::PyObject) = pyis(o, pynone) -export pyisnone - -function pynone_tryconvert(::Type{T}, o::PyObject) where {T} - tryconvert(T, nothing) -end diff --git a/src/number.jl b/src/number.jl deleted file mode 100644 index a919219e..00000000 --- a/src/number.jl +++ /dev/null @@ -1,41 +0,0 @@ -pyadd(o1, o2) = check(C.PyNumber_Add(pyobject(o1), pyobject(o2))) -pysub(o1, o2) = check(C.PyNumber_Subtract(pyobject(o1), pyobject(o2))) -pymul(o1, o2) = check(C.PyNumber_Multiply(pyobject(o1), pyobject(o2))) -pymatmul(o1, o2) = check(C.PyNumber_MatrixMultiply(pyobject(o1), pyobject(o2))) -pyfloordiv(o1, o2) = check(C.PyNumber_FloorDivide(pyobject(o1), pyobject(o2))) -pytruediv(o1, o2) = check(C.PyNumber_TrueDivide(pyobject(o1), pyobject(o2))) -pymod(o1, o2) = check(C.PyNumber_Remainder(pyobject(o1), pyobject(o2))) -pydivmod(o1, o2) = check(C.PyNumber_DivMod(pyobject(o1), pyobject(o2))) -pypow(o1, o2, o3=pynone) = check(C.PyNumber_Power(pyobject(o1), pyobject(o2), pyobject(o3))) -pyneg(o) = check(C.PyNumber_Negative(pyobject(o))) -pypos(o) = check(C.PyNumber_Positive(pyobject(o))) -pyabs(o) = check(C.PyNumber_Absolute(pyobject(o))) -pyinv(o) = check(C.PyNumber_Invert(pyobject(o))) -pylshift(o1, o2) = check(C.PyNumber_Lshift(pyobject(o1), pyobject(o2))) -pyrshift(o1, o2) = check(C.PyNumber_Rshift(pyobject(o1), pyobject(o2))) -pyand(o1, o2) = check(C.PyNumber_And(pyobject(o1), pyobject(o2))) -pyxor(o1, o2) = check(C.PyNumber_Xor(pyobject(o1), pyobject(o2))) -pyor(o1, o2) = check(C.PyNumber_Or(pyobject(o1), pyobject(o2))) -export pyadd, pysub, pymul, pymatmul, pyfloordiv, pytruediv, pymod, pydivmod, pypow, pyneg, pypos, pyabs, pyinv, pylshift, pyrshift, pyand, pyxor, pyor - -# in-place -pyiadd(o1, o2) = check(C.PyNumber_InPlaceAdd(pyobject(o1), pyobject(o2))) -pyisub(o1, o2) = check(C.PyNumber_InPlaceSubtract(pyobject(o1), pyobject(o2))) -pyimul(o1, o2) = check(C.PyNumber_InPlaceMultiply(pyobject(o1), pyobject(o2))) -pyimatmul(o1, o2) = check(C.PyNumber_InPlaceMatrixMultiply(pyobject(o1), pyobject(o2))) -pyifloordiv(o1, o2) = check(C.PyNumber_InPlaceFloorDivide(pyobject(o1), pyobject(o2))) -pyitruediv(o1, o2) = check(C.PyNumber_InPlaceTrueDivide(pyobject(o1), pyobject(o2))) -pyimod(o1, o2) = check(C.PyNumber_InPlaceRemainder(pyobject(o1), pyobject(o2))) -pyipow(o1, o2, o3=pynone) = check(C.PyNumber_InPlacePower(pyobject(o1), pyobject(o2), pyobject(o3))) -pyilshift(o1, o2) = check(C.PyNumber_InPlaceLshift(pyobject(o1), pyobject(o2))) -pyirshift(o1, o2) = check(C.PyNumber_InPlaceRshift(pyobject(o1), pyobject(o2))) -pyiand(o1, o2) = check(C.PyNumber_InPlaceAnd(pyobject(o1), pyobject(o2))) -pyixor(o1, o2) = check(C.PyNumber_InPlaceXor(pyobject(o1), pyobject(o2))) -pyior(o1, o2) = check(C.PyNumber_InPlaceOr(pyobject(o1), pyobject(o2))) -export pyiadd, pyisub, pyimul, pyimatmul, pyifloordiv, pyitruediv, pyimod, pyipow, pyilshift, pyirshift, pyiand, pyixor, pyior - -# conversions -pyint(o) = check(C.PyNumber_Long(pyobject(o))) -pyfloat(o) = check(C.PyNumber_Float(pyobject(o))) -pyindex(o) = check(C.PyNumber_Index(pyobject(o))) -export pyint, pyfloat, pyindex diff --git a/src/numpy.jl b/src/numpy.jl deleted file mode 100644 index c267ca68..00000000 --- a/src/numpy.jl +++ /dev/null @@ -1 +0,0 @@ -const pynumpy = pylazyobject(() -> pyimport("numpy")) diff --git a/src/object.jl b/src/object.jl deleted file mode 100644 index 3127b240..00000000 --- a/src/object.jl +++ /dev/null @@ -1,212 +0,0 @@ -### PYOBJECT - -mutable struct PyObject - ptr :: CPyPtr - get :: Function - function PyObject(::Val{:new}, src::Union{Function,Ptr}, borrowed::Bool=false) - if src isa Ptr - o = new(CPyPtr(src)) - borrowed && C.Py_IncRef(src) - else - o = new(CPyPtr(C_NULL), src) - end - finalizer(o) do o - if CONFIG.isinitialized - ptr = getfield(o, :ptr) - if ptr != C_NULL - with_gil() do - C.Py_DecRef(ptr) - end - end - end - setfield!(o, :ptr, CPyPtr(C_NULL)) - nothing - end - o - end -end -export PyObject - -""" - pynewobject(ptr::Ptr) - -Construct a `PyObject` from the new reference `ptr`. -""" -pynewobject(ptr::Ptr) = PyObject(Val(:new), ptr, false) - -""" - pyborrowedobject(ptr::Ptr) - -Construct a `PyObject` from the borrowed reference `ptr`. Its refcount is incremented. -""" -pyborrowedobject(ptr::Ptr) = PyObject(Val(:new), ptr, true) - -""" - pylazyobject(f::Function) - -Construct a `PyObject` lazily whose value is equal to `f()`, except evaluation of `f` is deferred until the object is used at run-time. - -Use these to construct module-level constants, such as `pynone`. -""" -pylazyobject(f::Function) = PyObject(Val(:new), f) - -function pyptr(o::PyObject) - # check if assigned - ptr = getfield(o, :ptr) - ptr == C_NULL || return ptr - # otherwise generate a value - val = getfield(o, :get)() - ptr = pyptr(val) - setfield!(o, :ptr, ptr) - C.Py_IncRef(ptr) - return ptr -end - -PyObject(o::PyObject) = o -PyObject(o) = PyObject(pyobject(o)) - -Base.convert(::Type{PyObject}, o::PyObject) = o -Base.convert(::Type{PyObject}, o) = PyObject(o) - -pyincref!(o::PyObject) = (C.Py_IncRef(pyptr(o)); o) - -Base.unsafe_convert(::Type{CPyPtr}, o::PyObject) = pyptr(o) - -### CONVERSIONS - -const pyobjecttype = pylazyobject(() -> pybuiltins.object) -export pyobjecttype - -pyobject(o::PyObject) = o -pyobject(args...; opts...) = pyobjecttype(args...; opts...) -pyobject(o::Nothing) = pynone -pyobject(o::Missing) = pynone -pyobject(o::Tuple) = pytuple(o) -pyobject(o::Pair) = pytuple(o) -pyobject(o::AbstractString) = pystr(o) -pyobject(o::Bool) = pybool(o) -pyobject(o::Integer) = pyint(o) -pyobject(o::Union{Float16,Float32,Float64}) = pyfloat(o) -pyobject(o::Union{Complex{Float16}, Complex{Float32}, Complex{Float64}}) = pycomplex(o) -pyobject(o::Rational) = pyfraction(o) -pyobject(o::AbstractRange{<:Integer}) = pyrange(o) -pyobject(o::DateTime) = pydatetime(o) -pyobject(o::Date) = pydate(o) -pyobject(o::Time) = pytime(o) -pyobject(o::IO) = pybufferedio(o) -pyobject(o::Function) = pyjl(o, Function) # functions are singleton types - this avoids compiling a separate type for each one -pyobject(o) = pyjl(o) -export pyobject - -### ABSTRACT OBJECT API - -pyrefcnt(o::PyObject) = C.Py_RefCnt(o) -export pyrefcnt - -pyis(o1::PyObject, o2::PyObject) = pyptr(o1) == pyptr(o2) -export pyis - -pyhasattr(o::PyObject, k::AbstractString) = check(Bool, C.PyObject_HasAttrString(o, k)) -pyhasattr(o::PyObject, k::Symbol) = pyhasattr(o, String(k)) -pyhasattr(o::PyObject, k) = check(Bool, C.PyObject_HasAttr(o, pyobject(k))) -export pyhasattr - -pygetattr(o::PyObject, k::AbstractString) = check(C.PyObject_GetAttrString(o, k)) -pygetattr(o::PyObject, k::Symbol) = pygetattr(o, String(k)) -pygetattr(o::PyObject, k) = check(C.PyObject_GetAttr(o, pyobject(k))) -export pygetattr - -pysetattr(o::PyObject, k::AbstractString, v) = check(Nothing, C.PyObject_SetAttrString(o, k, pyobject(v))) -pysetattr(o::PyObject, k::Symbol, v) = pysetattr(o, String(k), v) -pysetattr(o::PyObject, k, v) = check(Nothing, C.PyObject_SetAttr(o, pyobject(k), pyobject(v))) -export pysetattr - -pydelattr(o::PyObject, k::AbstractString) = check(Nothing, C.PyObject_DelAttrString(o, k)) -pydelattr(o::PyObject, k::Symbol) = pydelattr(o, String(k)) -pydelattr(o::PyObject, k) = check(Nothing, C.PyObject_DelAttr(o, pyobject(k))) -export pydelattr - -pyrichcompare(::Type{PyObject}, o1, o2, op::Cint) = check(C.PyObject_RichCompare(pyobject(o1), pyobject(o2), op)) -pyrichcompare(::Type{Bool}, o1, o2, op::Cint) = check(Bool, C.PyObject_RichCompareBool(pyobject(o1), pyobject(o2), op)) -pyrichcompare(o1, o2, op) = pyrichcompare(PyObject, o1, o2, op) -pyrichcompare(::Type{T}, o1, o2, ::typeof(< )) where {T} = pyrichcompare(T, o1, o2, C.Py_LT) -pyrichcompare(::Type{T}, o1, o2, ::typeof(<=)) where {T} = pyrichcompare(T, o1, o2, C.Py_LE) -pyrichcompare(::Type{T}, o1, o2, ::typeof(==)) where {T} = pyrichcompare(T, o1, o2, C.Py_EQ) -pyrichcompare(::Type{T}, o1, o2, ::typeof(!=)) where {T} = pyrichcompare(T, o1, o2, C.Py_NE) -pyrichcompare(::Type{T}, o1, o2, ::typeof(> )) where {T} = pyrichcompare(T, o1, o2, C.Py_GT) -pyrichcompare(::Type{T}, o1, o2, ::typeof(>=)) where {T} = pyrichcompare(T, o1, o2, C.Py_GE) -export pyrichcompare - -pyeq(args...) = pyrichcompare(args..., ==) -pyne(args...) = pyrichcompare(args..., !=) -pylt(args...) = pyrichcompare(args..., < ) -pyle(args...) = pyrichcompare(args..., <=) -pygt(args...) = pyrichcompare(args..., > ) -pyge(args...) = pyrichcompare(args..., >=) -export pyeq, pyne, pylt, pyle, pygt, pyge - -pyrepr(o) = check(C.PyObject_Repr(pyobject(o))) -pyrepr(::Type{String}, o) = pystr_asjuliastring(pyrepr(o)) -export pyrepr - -pyascii(o) = check(C.PyObject_ASCII(pyobject(o))) -pyascii(::Type{String}, o) = pystr_asjuliastring(pyascii(o)) -export pyascii - -pystr(o) = check(C.PyObject_Str(pyobject(o))) -pystr(::Type{String}, o) = pystr_asjuliastring(pystr(o)) -export pystr - -pybytes(o) = check(C.PyObject_Bytes(pyobject(o))) -export pybytes - -pyissubclass(o1::PyObject, o2::PyObject) = check(Bool, C.PyObject_IsSubclass(o1, o2)) -export pyissubclass - -pyisinstance(o::PyObject, t::PyObject) = check(Bool, C.PyObject_IsInstance(o, t)) -export pyisinstance - -pyhash(o) = check(C.PyObject_Hash(pyobject(o))) -export pyhash - -pytruth(o) = check(Bool, C.PyObject_IsTrue(pyobject(o))) -export pytruth - -pylen(o) = check(C.PyObject_Length(pyobject(o))) -export pylen - -pygetitem(o::PyObject, k) = check(C.PyObject_GetItem(o, pyobject(k))) -export pygetitem - -pysetitem(o::PyObject, k, v) = check(Nothing, C.PyObject_SetItem(o, pyobject(k), pyobject(v))) -export pysetitem - -pydelitem(o::PyObject, k) = check(Nothing, C.PyObject_DelItem(o, pyobject(k))) -export pydelitem - -pydir(o::PyObject) = check(C.PyObject_Dir(o)) -export pydir - -pyiter(o) = check(C.PyObject_GetIter(pyobject(o))) -pyiter(args...; opts...) = pybuiltins.iter(args...; opts...) -export pyiter - -pycall(f, args...; kwargs...) = - if !isempty(kwargs) - argso = pytuple_fromiter(args) - kwargso = pydict_fromstringiter(kwargs) - check(C.PyObject_Call(f, argso, kwargso)) - elseif !isempty(args) - argso = pytuple_fromiter(args) - check(C.PyObject_CallObject(f, argso)) - else - check(C.PyObject_CallObject(f, C_NULL)) - end -export pycall - -### MODULE OBJECTS - -const pymoduletype = pylazyobject(() -> pytype(pybuiltins)) -export pymoduletype - -pyismodule(o::PyObject) = pyisinstance(o, pymoduletype) diff --git a/src/datetime.jl b/src/old/datetime.jl similarity index 100% rename from src/datetime.jl rename to src/old/datetime.jl diff --git a/src/function.jl b/src/old/function.jl similarity index 100% rename from src/function.jl rename to src/old/function.jl diff --git a/src/old/julia.jl b/src/old/julia.jl new file mode 100644 index 00000000..b915cb0d --- /dev/null +++ b/src/old/julia.jl @@ -0,0 +1,1172 @@ +const PYJLGCCACHE = Dict{CPyPtr, Any}() + +function cpycatch(f, ::Type{T}=CPyPtr) where {T} + try + cpyreturn(T, f()) + catch err + if err isa PythonRuntimeError + # We restore Python errors. + # TODO: Is this the right behaviour? + pyerrrestore(err) + else + # Other (Julia) errors are raised as a JuliaException + bt = catch_backtrace() + val = try + pyjlbase((err, bt)) + catch + pynone + end + pyerrset(pyjlexception, val) + end + cpyerrval(T) + end +end + +cpyreturn(::Type{T}, x::T) where {T} = x +cpyreturn(::Type{CPyPtr}, x::PyObject) = CPyPtr(pyptr(pyincref!(x))) +cpyreturn(::Type{T}, x::Number) where {T<:Number} = convert(T, x) +cpyreturn(::Type{T}, x::Ptr) where {T<:Ptr} = T(x) + +cpyerrval(::Type{Nothing}) = nothing +cpyerrval(::Type{T}) where {T<:Number} = zero(T)-one(T) +cpyerrval(::Type{T}) where {T<:Ptr} = T(C_NULL) + +### jlexception + +const pyjlexception = pylazyobject() do + attrs = pydict() + attrs["__module__"] = "julia" + attrs["__str__"] = pymethod(o -> begin + val = o.args[0] + pyisnone(val) && return "Unknown" + err, bt = pyjlgetvalue(val) + return sprint(showerror, err) + end) + t = pytypetype("Exception", (pyexception,), attrs) + t +end +export pyjlexception + +### jlfunction + +@kwdef struct CPyJlFunctionObject + ob_base :: C.PyObject = C.PyObject() + call :: Ptr{Cvoid} = C_NULL +end + +cpyjlfunction_call(o::CPyPtr, args::CPyPtr, kwargs::CPyPtr) = ccall(UnsafePtr{CPyJlFunctionObject}(o).call[!], CPyPtr, (CPyPtr, CPyPtr), args, kwargs) +function cpyjlfunction_dealloc(o::CPyPtr) + delete!(PYJLGCCACHE, o) + ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (CPyPtr,), o) + nothing +end + +const pyjlfunctiontype = pylazyobject() do + # make the type + c = [] + t = cpynewtype!(c; + name = "julia.Function", + base = pyobjecttype, + basicsize = sizeof(CPyJlFunctionObject), + flags = C.Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? C.Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), + call = @cfunction(cpyjlfunction_call, CPyPtr, (CPyPtr, CPyPtr, CPyPtr)), + dealloc = @cfunction(cpyjlfunction_dealloc, Cvoid, (CPyPtr,)), + ) + # put into a 0-dim array and take a pointer + ta = fill(t) + ptr = pointer(ta) + # ready the type + check(C.PyType_Ready(ptr)) + # success + PYJLGCCACHE[CPyPtr(ptr)] = push!(c, ta) + pyborrowedobject(ptr) +end +export pyjlfunctiontype + +struct cpyjlfunction_call_impl{F} + f :: F +end +(f::cpyjlfunction_call_impl)(_args::CPyPtr, _kwargs::CPyPtr) = cpycatch() do + if _kwargs != C_NULL + args = pyborrowedobject(_args) + kwargs = Dict{Symbol,PyObject}(Symbol(string(k)) => v for (k,v) in pyborrowedobject(_kwargs).items()) + pyobject(f.f(args...; kwargs...)) + elseif _args != C_NULL + args = pyborrowedobject(_args) + nargs = length(args) + if nargs == 0 + pyobject(f.f()) + elseif nargs == 1 + pyobject(f.f(args[0])) + elseif nargs == 2 + pyobject(f.f(args[0], args[1])) + elseif nargs == 3 + pyobject(f.f(args[0], args[1], args[2])) + else + pyobject(f.f(args...)) + end + else + pyobject(f.f()) + end +end + +function pyjlfunction(f) + # allocate an object + o = check(C._PyObject_New(pyjlfunctiontype)) + # set value + p = UnsafePtr{CPyJlFunctionObject}(pyptr(o)) + c = [] + p.call[] = cacheptr!(c, @cfunction($(cpyjlfunction_call_impl(f)), CPyPtr, (CPyPtr, CPyPtr))) + PYJLGCCACHE[pyptr(o)] = c + # done + return o +end +export pyjlfunction + +### jlbasevalue + +@kwdef struct CPyJlValueObject{T} + ob_base :: C.PyObject = C.PyObject() + value :: Ptr{Cvoid} = C_NULL + weaklist :: CPyPtr = C_NULL +end +const CPyJlPtr{T} = Ptr{CPyJlValueObject{T}} + +function cpyjlvalue_dealloc(o::CPyPtr) + delete!(PYJLGCCACHE, o) + UnsafePtr{CPyJlValueObject{Any}}(o).weaklist[!] == C_NULL || C.PyObject_ClearWeakRefs(o) + ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (CPyPtr,), o) + nothing +end + +function cpyjlvalue_new(t::CPyPtr, args::CPyPtr, kwargs::CPyPtr) + o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], CPyPtr, (CPyPtr, C.Py_ssize_t), t, 0) + if o != C_NULL + p = UnsafePtr{CPyJlValueObject{Any}}(o) + p.value[] = C_NULL + p.weaklist[] = C_NULL + end + o +end + +cpyjlvalue_get_buffer(_o::CPyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) = cpycatch(Cint) do + o = pyborrowedobject(_o) + t = pytype(o) + if pyhasattr(t, "__jl_enable_buffer__") && pytruth(t.__jl_enable_buffer__) + pyjl_get_buffer(o, buf, flags) + else + pythrow(pybuffererror("Buffer protocol not supported by '$(t.__name__)'")) + end +end + +function cpyjlvalue_release_buffer(_o::CPyPtr, buf::Ptr{C.Py_buffer}) + delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[]) + nothing +end + +const pyjlbasetype = pylazyobject() do + # make the type + c = [] + t = cpynewtype!(c; + name = "julia.ValueBase", + basicsize = sizeof(CPyJlValueObject{Any}), + new = @cfunction(cpyjlvalue_new, CPyPtr, (CPyPtr, CPyPtr, CPyPtr)), + dealloc = @cfunction(cpyjlvalue_dealloc, Cvoid, (CPyPtr,)), + flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG | (CONFIG.isstackless ? C.Py_TPFLAGS_HAVE_STACKLESS_EXTENSION : 0x00), + weaklistoffset = fieldoffset(CPyJlValueObject{Any}, 3), + getattro = C.pyglobal(:PyObject_GenericGetAttr), + setattro = C.pyglobal(:PyObject_GenericSetAttr), + doc = "A Julia value with no semantics.", + as_buffer = (get=@cfunction(cpyjlvalue_get_buffer, Cint, (CPyPtr, Ptr{C.Py_buffer}, Cint)), release=@cfunction(cpyjlvalue_release_buffer, Cvoid, (CPyPtr, Ptr{C.Py_buffer}))), + ) + # put into a 0-dim array and take a pointer + ta = fill(t) + ptr = pointer(ta) + # ready the type + check(C.PyType_Ready(ptr)) + # success + PYJLGCCACHE[CPyPtr(ptr)] = push!(c, ta) + pyborrowedobject(ptr) +end +export pyjlbasetype + +function pyjlnewvalue(x, t::PyObject) + pyissubclass(t, pyjlbasetype) || error("Expecting a subtype of 'julia.ValueBase'") + # allocate + o = t() + # set value + p = UnsafePtr{CPyJlValueObject{Any}}(pyptr(o)) + p.value[], PYJLGCCACHE[pyptr(o)] = pointer_from_obj(x) + # done + return o +end + +pyjlbase(x) = pyjlnewvalue(x, pyjlbasetype) +export pyjlbase + +pyisjl(o::PyObject) = pyisinstance(o, pyjlbasetype) +export pyisjl + +cpyjlgetvalue(o::CPyJlPtr{T}) where {T} = Base.unsafe_pointer_to_objref(UnsafePtr(o).value[!]) :: T +cpyjlgetvalue(o::Ptr) = cpyjlgetvalue(CPyJlPtr{Any}(o)) + +function pyjlgetvalue(o::PyObject, ::Type{T}=Any) where {T} + pyisinstance(o, pyjlbasetype) || pythrow(pytypeerror("Expecting a Python 'julia.ValueBase'")) + ptr = UnsafePtr(CPyJlPtr{Any}(pyptr(o))).value[!] + ptr == C_NULL && pythrow(pyvalueerror("Value is NULL")) + Base.unsafe_pointer_to_objref(ptr)::T +end +export pyjlgetvalue + +### jlrawvalue + +const PYJLRAWTYPES = Dict{Type,PyObject}() + +pyjlrawtype(::Type{T}) where {T} = get!(PYJLRAWTYPES, T) do + base = pyjl_supertype(T)===nothing ? pyjlbasetype : pyjlrawtype(pyjl_supertype(T)) + attrs = pydict() + attrs["__slots__"] = pytuple() + attrs["__module__"] = "julia" + attrs["__repr__"] = pymethod(o -> "") + attrs["__str__"] = pymethod(o -> string(pyjlgetvalue(o, T))) + attrs["__doc__"] = """ + A Julia '$T' with basic Julia semantics. + """ + # Logic + # Note that comparisons use isequal and isless, so that hashing is supported + attrs["__eq__"] = pymethod((o, x) -> pybool(isequal(pyjlgetvalue(o, T), pyconvert(Any, x)))) + attrs["__lt__"] = pymethod((o, x) -> pybool(isless(pyjlgetvalue(o, T), pyconvert(Any, x)))) + attrs["__bool__"] = pymethod(o -> (v=pyjlgetvalue(o, T); v isa Bool ? pybool(v) : pythrow(pytypeerror("Only 'Bool' can be tested for truthyness")))) + attrs["__hash__"] = pymethod(o -> pyint(hash(pyjlgetvalue(o, T)))) + # Containers + attrs["__contains__"] = pymethod((o, v) -> pybool(pyconvert(Any, v) in pyjlgetvalue(o, T))) + attrs["__getitem__"] = pymethod((o, i) -> o.__jl_wrap_result(pyjlraw(getindex(pyjlgetvalue(o, T), (pyistuple(i) ? [pyconvert(Any, j) for j in i] : [pyconvert(Any, i)])...)))) + attrs["__setitem__"] = pymethod((o, i, v) -> (setindex!(pyjlgetvalue(o, T), pyconvert(Any, v), (pyistuple(i) ? [pyconvert(Any, j) for j in i] : [pyconvert(Any, i)])...); pynone)) + attrs["__iter__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(Iterator(pyjlgetvalue(o, T), nothing)))) + if T <: Iterator + attrs["__next__"] = pymethod(_o -> begin + o = pyjlgetvalue(_o, T) + if o.st === nothing + it = iterate(o.val) + else + it = iterate(o.val, something(o.st)) + end + if it === nothing + pythrow(pystopiteration()) + else + x, st = it + o.st = Some(st) + return pyjlraw(x) + end + end) + end + # Function call + # TODO: keywords + attrs["__call__"] = pymethod((o, args...; kwargs...) -> begin + f = pyjlgetvalue(o, T) + jargs = map(x->pyconvert(Any, x), args) + r = if isempty(kwargs) + f(jargs...) + else + jkwargs = Tuple(k => pyconvert(Any, v) for (k,v) in kwargs) + f(jargs...; jkwargs...) + end + o.__jl_wrap_result(pyjlraw(r)) + end) + # Attributes + attrs["__dir__"] = pymethod(o -> begin + d = pyobjecttype.__dir__(o) + d.extend(pylist([pyjl_attrname_jl2py(string(k)) for k in propertynames(pyjlgetvalue(o, T), false)])) + d + end) + attrs["__getattr__"] = pymethod((_o, _k) -> begin + # first do a generic lookup + _x = C.PyObject_GenericGetAttr(_o, _k) + if _x != C_NULL + return pynewobject(_x) + elseif !pyerroccurred(pyattributeerror) + pythrow() + end + st = pyerrfetch() + # try to get a julia property with this name + o = pyjlgetvalue(_o, T) + k = Symbol(pyjl_attrname_py2jl(pystr_asjuliastring(_k))) + try + return _o.__jl_wrap_result(pyjlraw(getproperty(o, k))) + catch err + if (err isa UndefVarError) || (err isa ErrorException && occursin("has no field", err.msg)) + throw(PythonRuntimeError(st...)) + else + rethrow() + end + end + end) + attrs["__setattr__"] = pymethod((_o, _k, _v) -> begin + # first do a generic lookup + _x = C.PyObject_GenericSetAttr(_o, _k, _v) + if _x != -1 + return pynone + elseif !pyerroccurred(pyattributeerror) + pythrow() + end + st = pyerrfetch() + # try to set a julia property with this name + o = pyjlgetvalue(_o, T) + k = Symbol(pyjl_attrname_py2jl(pystr_asjuliastring(_k))) + v = pyconvert(Any, _v) + try + setproperty!(o, k, v) + return pynone + catch err + if (err isa UndefVarError) || (err isa ErrorException && occursin("has no field", err.msg)) + throw(PythonRuntimeError(st...)) + else + rethrow() + end + end + end) + # Arithmetic + attrs["__add__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) + pyconvert(Any, x)))) + attrs["__sub__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) - pyconvert(Any, x)))) + attrs["__mul__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) * pyconvert(Any, x)))) + attrs["__truediv__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) / pyconvert(Any, x)))) + attrs["__floordiv__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(fld(pyjlgetvalue(o, T), pyconvert(Any, x))))) + attrs["__mod__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(mod(pyjlgetvalue(o, T), pyconvert(Any, x))))) + attrs["__pow__"] = pymethod((o, x, m=pynone) -> o.__jl_wrap_result(pyjlraw(pyisnone(m) ? pyjlgetvalue(o, T) ^ pyconvert(Any, x) : powermod(pyjlgetvalue(o, T), pyconvert(Any, x), pyjlgetvalue(m))))) + attrs["__lshift__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) << pyconvert(Any, x)))) + attrs["__rshift__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) >> pyconvert(Any, x)))) + attrs["__and__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) & pyconvert(Any, x)))) + attrs["__xor__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) ⊻ pyconvert(Any, x)))) + attrs["__or__"] = pymethod((o, x) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T) | pyconvert(Any, x)))) + attrs["__neg__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(-pyjlgetvalue(o, T)))) + attrs["__pos__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(+pyjlgetvalue(o, T)))) + attrs["__abs__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(abs(pyjlgetvalue(o, T))))) + attrs["__invert__"] = pymethod(o -> o.__jl_wrap_result(pyjlraw(~pyjlgetvalue(o, T)))) + attrs["__complex__"] = pymethod(o -> pycomplex(convert(Complex{Cdouble}, pyjlgetvalue(o, T)))) + attrs["__float__"] = pymethod(o -> pyfloat(convert(Cdouble, pyjlgetvalue(o, T)))) + attrs["__int__"] = attrs["__index__"] = pymethod(o -> pyint(convert(Integer, pyjlgetvalue(o, T)))) + # Julia-specific + attrs["__jl_getfield"] = pymethod((o, k) -> o.__jl_wrap_result(pyjlraw(getfield(pyjlgetvalue(o, T), pyjlraw_propertyname(k))))) + attrs["__jl_getprop"] = pymethod((o, k) -> o.__jl_wrap_result(pyjlraw(getproperty(pyjlgetvalue(o, T), pyjlraw_propertyname(k))))) + attrs["__jl_setfield"] = pymethod((o, k, v) -> (setfield!(pyjlgetvalue(o, T), pyjlraw_propertyname(k), pyjlgetvalue(v)); pynone)) + attrs["__jl_setprop"] = pymethod((o, k, v) -> (setproperty!(pyjlgetvalue(o, T), pyjlraw_propertyname(k), pyjlgetvalue(v)); pynone)) + attrs["__jl_prop"] = pymethod((o, k, v=nothing) -> v===nothing ? o.__jl_getprop(k) : o.__jl_setprop(k, v)) + attrs["__jl_field"] = pymethod((o, k, v=nothing) -> v===nothing ? o.__jl_getfield(k) : o.__jl_setfield(k, v)) + attrs["__jl_typeof"] = isconcretetype(T) ? pymethod(o -> o.__jl_wrap_result(pyjlraw(T))) : pymethod(o -> o.__jl_wrap_result(pyjlraw(typeof(pyjlgetvalue(o, T))))) + attrs["__jl_wrap_result"] = pymethod((o, v) -> v.__jl_raw()) + attrs["__jl_pyobject"] = pymethod(o -> pyobject(pyjlgetvalue(o, T))) + attrs["__jl_raw"] = pymethod(o -> o) + attrs["__jl_curly"] = pymethod((o, args...) -> o.__jl_wrap_result(pyjlraw(pyjlgetvalue(o, T){map(x->pyconvert(Any, x), args)...}))) + # Done + pytypetype("RawValue[$T]", (base,), attrs) +end +export pyjlrawtype + +pyjlraw_propertyname(k::PyObject) = pyisstr(k) ? Symbol(pystr_asjuliastring(k)) : pyisint(k) ? pyint_tryconvert(Int, k) : pyconvert(Any, k) + +pyjlraw(x, ::Type{T}) where {T} = x isa T ? pyjlnewvalue(x, pyjlrawtype(T)) : error("expecting a `$T`") +pyjlraw(x) = pyjlraw(x, typeof(x)) +export pyjlraw + +### jlvalue + +const PYJLTYPES = Dict{Type,PyObject}() + +function pyjl_supertypes(::Type{T}) where {T} + r = [] + S = T + while S !== nothing + push!(r, S) + S = pyjl_supertype(S) + end + r +end + +pyjl_supertype(::Type{T}) where {T} = supertype(T) +pyjl_supertype(::Type{Any}) = nothing +pyjl_supertype(::Type{Type{T}}) where {T} = Any +pyjl_supertype(::Type{Type}) = Any +pyjl_supertype(::Type{DataType}) = Type +pyjl_supertype(::Type{UnionAll}) = Type +pyjl_supertype(::Type{Union}) = Type + +pyjl_valuetype(::Type{T}) where {T} = T + +abstract type PyJlSubclass{S} end + +pyjl_supertype(::Type{T}) where {S, T<:PyJlSubclass{S}} = + if supertype(T) <: PyJlSubclass + if supertype(supertype(T)) <: PyJlSubclass + supertype(T) + else + S + end + else + error("cannot instantiate `PyJlSubclass`") + end +pyjl_supertype(::Type{T}) where {T<:PyJlSubclass} = error() + +pyjl_valuetype(::Type{T}) where {S, T<:PyJlSubclass{S}} = pyjl_valuetype(S) +pyjl_valuetype(::Type{T}) where {T<:PyJlSubclass} = error() + +pyjltype(::Type{T}) where {T} = get!(PYJLTYPES, T) do + # Bases + Ss = pyjl_supertypes(T) + V = pyjl_valuetype(T) + bases = [pyjlrawtype(V)] + length(Ss) == 1 || pushfirst!(bases, pyjltype(Ss[2])) + mixin = pyjl_mixin(T) + pyisnone(mixin) || push!(bases, mixin) + + # Attributes + attrs = Dict{String,PyObject}() + attrs["__module__"] = "julia" + attrs["__slots__"] = pytuple() + attrs["__jl_wrap_result"] = pymethod((o, x) -> x.__jl_pyobject()) + attrs["__jl_raw"] = pymethod(o -> pyjlraw(pyjlgetvalue(o, V))) + for S in reverse(Ss) + V <: pyjl_valuetype(S) || error() + pyjl_addattrs(attrs, S, V) + end + + # Make the type + t = pytypetype("Value[$T]", pytuple_fromiter(bases), pydict_fromstringiter(attrs)) + + # Register an abstract base class + abc = pyjl_abc(T) + pyisnone(abc) || abc.register(t) + + # Done + return t +end +export pyjltype + +pyjl(x, ::Type{T}) where {T} = x isa pyjl_valuetype(T) ? pyjlnewvalue(x, pyjltype(T)) : error("expecting a `$(pyjl_valuetype(T))`") +pyjl(x) = pyjl(x, typeof(x)) +export pyjl + +### Any + +pyjl_abc(::Type) = pynone +pyjl_mixin(::Type) = pynone + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Any, V<:T} + t["__repr__"] = pymethod(o -> "") + t["__str__"] = pymethod(o -> string(pyjlgetvalue(o, V))) + t["__doc__"] = """ + A Julia '$T' with Python semantics. + """ + if hasmethod(length, Tuple{V}) + t["__len__"] = pymethod(o -> length(pyjlgetvalue(o, V))) + end + if hasmethod(in, Tuple{Union{}, V}) + t["__contains__"] = pymethod((_o, _x) -> begin + o = pyjlgetvalue(_o, V) + x = pytryconvert_element(o, _x) + x === PyConvertFail() ? false : in(x, o) + end) + end + if hasmethod(reverse, Tuple{V}) + t["__reversed__"] = pymethod(o -> pyjl(reverse(pyjlgetvalue(o, V)))) + end + if hasmethod(iterate, Tuple{V}) + t["__iter__"] = pymethod(o -> pyjl(Iterator(pyjlgetvalue(o, V), nothing))) + end + if hasmethod(getindex, Tuple{V, Union{}}) + t["__getitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert_indices(o, _k) + if k === PyConvertFail() + pythrow(pytypeerror("invalid index")) + end + pyobject(o[k...]) + end) + end + if hasmethod(setindex!, Tuple{V, Union{}, Union{}}) + t["__setitem__"] = pymethod((_o, _k, _v) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert_indices(o, _k) + k === PyConvertFail && pythrow(pytypeerror("invalid index")) + v = pytryconvert_value(o, _v, k...) + v === PyConvertFail && pythrow(pytypeerror("invalid value")) + o[k...] = v + pynone + end) + end + # Comparisons + t["__eq__"] = pymethod((o,x) -> pyjlgetvalue(o, V) == pyconvert(Any, x)) + t["__ne__"] = pymethod((o,x) -> pyjlgetvalue(o, V) != pyconvert(Any, x)) + t["__le__"] = pymethod((o,x) -> pyjlgetvalue(o, V) <= pyconvert(Any, x)) + t["__lt__"] = pymethod((o,x) -> pyjlgetvalue(o, V) < pyconvert(Any, x)) + t["__ge__"] = pymethod((o,x) -> pyjlgetvalue(o, V) >= pyconvert(Any, x)) + t["__gt__"] = pymethod((o,x) -> pyjlgetvalue(o, V) > pyconvert(Any, x)) + # MIME display + for (mime, method) in ( + (MIME"text/html"(), "_repr_html_"), + (MIME"text/markdown"(), "_repr_markdown_"), + (MIME"text/json"(), "_repr_json_"), + (MIME"application/javascript"(), "_repr_javascript_"), + (MIME"application/pdf"(), "_repr_pdf_"), + (MIME"image/jpeg"(), "_repr_jpeg_"), + (MIME"image/png"(), "_repr_png_"), + (MIME"image/svg+xml"(), "_repr_svg_"), + (MIME"text/latex"(), "_repr_latex_")) + + t[method] = pymethod(_o -> begin + o = pyjlgetvalue(_o, V) + if showable(mime, o) + io = IOBuffer() + show(io, mime, o) + r = pybytes(take!(io)) + istextmime(mime) ? r.decode("utf8") : r + else + pynone + end + end) + end +end + +pyjl_attrname_py2jl(x::AbstractString) = + replace(x, r"_[b]+$" => s -> "!"^(length(s)-1)) + +pyjl_attrname_jl2py(x::AbstractString) = + replace(x, r"!+$" => s -> "_" * "b"^(length(s))) + +### Nothing & Missing (falsy) + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Nothing,Missing}, V<:T} + t["__bool__"] = pymethod(o -> pyfalse) +end + +### Module + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Module, V<:T} + t["__dir__"] = pymethod(_o -> begin + d = pyjltype(Any).__dir__(_o) + o = pyjlgetvalue(_o, V) + for n in names(o, all=true, imported=true) + d.append(pyjl_attrname_jl2py(string(n))) + end + for m in ccall(:jl_module_usings, Any, (Any,), o)::Vector + for n in names(m) + d.append(pyjl_attrname_jl2py(string(n))) + end + end + return d + end) +end + +### Type + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Type, V<:T} + t["curly"] = pymethod((o, args...) -> o.__jl_curly(args...)) +end + +### Iterator (as Iterator) + +mutable struct Iterator{T} + val :: T + st :: Union{Nothing, Some} +end + +pyjl_mixin(::Type{T}) where {T<:Iterator} = pyiteratorabc + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Iterator, V<:T} + t["__iter__"] = pymethod(o -> o) + t["__next__"] = pymethod(_o -> begin + o = pyjlgetvalue(_o, V) + if o.st === nothing + it = iterate(o.val) + else + it = iterate(o.val, something(o.st)) + end + if it === nothing + pythrow(pystopiteration()) + else + x, st = it + o.st = Some(st) + return pyobject(x) + end + end) +end + +### Number + +pyjl_mixin(::Type{T}) where {T<:Number} = pynumbersmodule.Number +pyjl_mixin(::Type{T}) where {T<:Complex} = pynumbersmodule.Complex +pyjl_mixin(::Type{T}) where {T<:Real} = pynumbersmodule.Real +pyjl_mixin(::Type{T}) where {T<:Rational} = pynumbersmodule.Rational +pyjl_mixin(::Type{T}) where {T<:Integer} = pynumbersmodule.Integral + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Number, V<:T} + t["__bool__"] = pymethod(o -> pybool(!iszero(pyjlgetvalue(o, V)))) + t["__pos__"] = pymethod(o -> pyobject(+pyjlgetvalue(o, V))) + t["__neg__"] = pymethod(o -> pyobject(-pyjlgetvalue(o, V))) +end + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Complex,Real}, V<:T} + t["real"] = pyproperty(o -> pyobject(real(pyjlgetvalue(o, V)))) + t["imag"] = pyproperty(o -> pyobject(imag(pyjlgetvalue(o, V)))) + t["conjugate"] = pymethod(o -> pyobject(conj(pyjlgetvalue(o, V)))) + t["__abs__"] = pymethod(o -> pyobject(abs(pyjlgetvalue(o, V)))) + t["__complex__"] = pymethod(o -> pycomplex(pyjlgetvalue(o, V))) + if V<:Real + t["__float__"] = pymethod(o -> pyfloat(pyjlgetvalue(o, V))) + t["__trunc__"] = pymethod(o -> pyint(trunc(BigInt, pyjlgetvalue(o, V)))) + t["__round__"] = pymethod((o,n=pynone) -> pyisnone(n) ? pyint(round(BigInt, pyjlgetvalue(o, V))) : pyjl(round(pyjlgetvalue(o, V), digits=pyconvert(Int, n)))) + t["__floor__"] = pymethod(o -> pyint(floor(BigInt, pyjlgetvalue(o, V)))) + t["__ceil__"] = pymethod(o -> pyint(ceil(BigInt, pyjlgetvalue(o, V)))) + end +end + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Integer,Rational}, V<:T} + t["numerator"] = pyproperty(o -> pyobject(numerator(pyjlgetvalue(o, V)))) + t["denominator"] = pyproperty(o -> pyobject(denominator(pyjlgetvalue(o, V)))) + if V<:Integer + t["__int__"] = pymethod(o -> pyint(pyjlgetvalue(o, V))) + t["__invert__"] = pymethod(o -> pyobject(~pyjlgetvalue(o, V))) + end +end + +### Dict (as Mapping) + +pyjl_mixin(::Type{T}) where {T<:AbstractDict} = pymutablemappingabc + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractDict, V<:T} + t["__iter__"] = pymethod(o -> pyjl(Iterator(keys(pyjlgetvalue(o, V)), nothing))) + t["__getitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert(keytype(o), _k) + if k === PyConvertFail() || !haskey(o, k) + pythrow(pykeyerror(_k)) + end + pyobject(o[k]) + end) + t["__setitem__"] = pymethod((_o, _k, _v) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert(keytype(o), _k) + if k === PyConvertFail() + pythrow(pykeyerror(_k)) + end + v = pytryconvert(valtype(o), _v) + v === PyConvertFail() && pythrow(pytypeerror("invalid value of type '$(pytype(_v).__name__)'")) + o[k] = v + pynone + end) + t["__delitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert(keytype(o), _k) + if k === PyConvertFail() || !haskey(o, k) + pythrow(pykeyerror(_k)) + end + delete!(o, k) + pynone + end) + t["__contains__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pytryconvert(keytype(o), _k) + k === PyConvertFail() ? false : haskey(o, k) + end) + t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) +end + +### Set (as Set) + +pyjl_mixin(::Type{T}) where {T<:AbstractSet} = pymutablesetabc + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractSet, V<:T} + t["add"] = pymethod((_o, _v) -> begin + o = pyjlgetvalue(_o, V) + v = pytryconvert(eltype(o), _v) + v === PyConvertFail() && pythrow(pytypeerror("Invalid value of type '$(pytype(v).__name__)'")) + push!(o, v) + pynone + end) + t["discard"] = pymethod((_o, _v) -> begin + o = pyjlgetvalue(_o, V) + v = pytryconvert(eltype(o), _v) + v === PyConvertFail() || delete!(o, v) + pynone + end) + t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) +end + +### Array (as Collection) +### Vector (as Sequence) + +pyjl_mixin(::Type{T}) where {T<:AbstractArray} = pycollectionabc +pyjl_mixin(::Type{T}) where {T<:AbstractVector} = pymutablesequenceabc + +function pyjl_axisidx(ax, k::PyObject) + # slice + if pyisslice(k) + error("slicing not implemented") + end + # convert to int + if pyisint(k) + # nothing to do + elseif pyhasattr(k, "__index__") + k = k.__index__() + else + pythrow(pytypeerror("Indices must be 'int' or 'slice', not '$(pytype(k).__name__)'")) + end + # convert to julia int + i = check(C.PyLong_AsLongLong(k), true) + # negative indexing + j = i<0 ? i+length(ax) : i + # bounds check + 0 ≤ j < length(ax) || pythrow(pyindexerror("Index out of range")) + # adjust for zero-up indexing + j + first(ax) +end + +function pyjl_arrayidxs(o::AbstractArray{T,N}, k::PyObject) where {T,N} + if pyistuple(k) + N == pylen(k) && return ntuple(i -> pyjl_axisidx(axes(o, i), k[i-1]), N) + else + N == 1 && return (pyjl_axisidx(axes(o, 1), k),) + end + pythrow(pytypeerror("Expecting a tuple of $N indices")) +end + +pyjl_isbufferabletype(::Type{T}) where {T} = + T in (Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}, Bool, Ptr{Cvoid}) +pyjl_isbufferabletype(::Type{T}) where {T<:Tuple} = + isconcretetype(T) && Base.allocatedinline(T) && all(pyjl_isbufferabletype, fieldtypes(T)) +pyjl_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = + pyjl_isbufferabletype(T) + +pyjl_isarrayabletype(::Type{T}) where {T} = + T in (UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64, Bool, Float16, Float32, Float64, Complex{Float16}, Complex{Float32}, Complex{Float64}) + +islittleendian() = Base.ENDIAN_BOM == 0x04030201 ? true : Base.ENDIAN_BOM == 0x01020304 ? false : error() + +function pytypestrformat(::Type{T}) where {T} + c = islittleendian() ? '<' : '>' + T == Int8 ? ("$(c)i1", pynone) : + T == UInt8 ? ("$(c)u1", pynone) : + T == Int16 ? ("$(c)i2", pynone) : + T == UInt16 ? ("$(c)u2", pynone) : + T == Int32 ? ("$(c)i4", pynone) : + T == UInt32 ? ("$(c)u4", pynone) : + T == Int64 ? ("$(c)i8", pynone) : + T == UInt64 ? ("$(c)u8", pynone) : + T == Float16 ? ("$(c)f2", pynone) : + T == Float32 ? ("$(c)f4", pynone) : + T == Float64 ? ("$(c)f8", pynone) : + error("not implemented") +end + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:AbstractArray, V<:T} + if hasmethod(pointer, Tuple{V}) && hasmethod(strides, Tuple{V}) + if pyjl_isbufferabletype(eltype(V)) + t["__jl_enable_buffer__"] = true + end + if pyjl_isarrayabletype(eltype(V)) + t["__array_interface__"] = pyproperty(_o -> begin + o = pyjlgetvalue(_o, V) + typestr, descr = pytypestrformat(eltype(o)) + pydict( + shape = size(o), + typestr = typestr, + descr = descr, + data = (UInt(pointer(o)), !ismutablearray(o)), + strides = strides(o) .* Base.aligned_sizeof(eltype(o)), + version = 3, + ) + end) + end + end + if V <: PyObjectArray + t["__array_interface__"] = pyproperty(_o -> begin + o = pyjlgetvalue(_o, V) + pydict( + shape = size(o), + typestr = "O", + data = (UInt(pointer(o.ptrs)), false), + strides = strides(o.ptrs) .* sizeof(CPyPtr), + version = 3, + ) + end) + end + t["ndim"] = pyproperty(o -> pyint(ndims(pyjlgetvalue(o, V)))) + t["shape"] = pyproperty(o -> pytuple(map(pyint, size(pyjlgetvalue(o, V))))) + t["__array__"] = pymethod(o -> pyhasattr(o, "__array_interface__") ? pynumpy.asarray(o) : pynumpy.asarray(PyObjectArray(pyjlgetvalue(o, V)))) + t["__getitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pyjl_arrayidxs(o, _k) + pyobject(o[k...]) + end) + t["__setitem__"] = pymethod((_o, _k, _v) -> begin + o = pyjlgetvalue(_o, V) + k = pyjl_arrayidxs(o, _k) + v = pytryconvert(eltype(o), _v) + v === PyConvertFail() && pythrow(pytypeerror("Cannot assign value of type '$(pytype(_v).__name__)'")) + o[k...] = v + pynone + end) + t["copy"] = pymethod(o -> pyjl(copy(pyjlgetvalue(o, V)))) + if V <: AbstractVector + t["__delitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + k = pyjl_arrayidxs(o, _k) + deleteat!(o, k...) + pynone + end) + t["insert"] = pymethod((_o, _k, _v) -> begin + o = pyjlgetvalue(_o, V) + ax = axes(o, 1) + k = pyjl_axisidx(first(ax):(last(ax)+1), _k) + v = pytryconvert(eltype(o), _v) + v === PyConvertFail() && pythrow(pytypeerror("Cannot assign value of type '$(pytype(_v).__name__)'")) + insert!(o, k, v) + pynone + end) + t["sort"] = pymethod((_o; reverse=pyfalse, key=pynone) -> begin + o = pyjlgetvalue(_o) + rev = pytruth(reverse) + by = pyisnone(key) ? identity : key + sort!(o, rev=rev, by=by) + pynone + end) + t["clear"] = pymethod(o -> (empty!(pyjlgetvalue(o, V)); pynone)) + end +end + +### Tuple & NamedTuple (as Sequence) + +pyjl_mixin(::Type{T}) where {T<:Union{Tuple,NamedTuple}} = pysequenceabc + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:Union{Tuple,NamedTuple}, V<:T} + t["__getitem__"] = pymethod((_o, _k) -> begin + o = pyjlgetvalue(_o, V) + if o isa NamedTuple && pyisstr(_k) + k = Symbol(string(_k)) + haskey(o, k) || pythrow(pykeyerror(_k)) + else + k = pyjl_axisidx(1:length(o), _k) + end + pyobject(o[k]) + end) +end + +### IO (as IOBase) + +pyjl_abc(::Type{T}) where {T<:IO} = pyiomodule.IOBase + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:IO, V} + t["close"] = if hasmethod(close, Tuple{V}) + pymethod(o -> (close(pyjlgetvalue(o, V)); pynone)) + else + pymethod(o -> pythrow(pyiounsupportedoperation("close"))) + end + t["closed"] = if hasmethod(isopen, Tuple{V}) + pyproperty(o -> pybool(!isopen(pyjlgetvalue(o, V)))) + else + pyproperty(o -> pythrow(pyiounsupportedoperation("closed"))) + end + t["fileno"] = if hasmethod(fd, Tuple{V}) + pymethod(o -> pyint(fd(pyjlgetvalue(o, V)))) + else + pymethod(o -> pythrow(pyiounsupportedoperation("fileno"))) + end + t["flush"] = pymethod(o -> (flush(pyjlgetvalue(o, V)); pynone)) + t["isatty"] = V<:Base.TTY ? pymethod(o -> pybool(true)) : pymethod(o -> pybool(false)) + t["readable"] = if hasmethod(isreadable, Tuple{V}) + pymethod(o -> pybool(isreadable(pyjlgetvalue(o, V)))) + else + pymethod(o -> pythrow(pyiounsupportedoperation("readable"))) + end + t["writable"] = if hasmethod(iswritable, Tuple{V}) + pymethod(o -> pybool(iswritable(pyjlgetvalue(o, V)))) + else + pymethod(o -> pythrow(pyiounsupportedoperation("writable"))) + end + t["tell"] = if hasmethod(position, Tuple{V}) + pymethod(o -> pyint(position(pyjlgetvalue(o, V)))) + else + pymethod(o -> pythrow(pyiounsupportedoperation("tell"))) + end + t["writelines"] = pymethod((o, lines) -> begin + wr = o.write + for line in lines + wr(line) + end + pynone + end) + t["seekable"] = if hasmethod(position, Tuple{V}) && hasmethod(seek, Tuple{V,Int}) && hasmethod(truncate, Tuple{V,Int}) + pymethod(o -> pybool(true)) + else + pymethod(o -> pybool(false)) + end + t["truncate"] = if hasmethod(truncate, Tuple{V, Int}) + pymethod((_o, _n=pynone) -> begin + o = pyjlgetvalue(_o, V) + n = pyisnone(_n) ? position(o) : pyconvert(Int, _n) + truncate(o, n) + pyint(n) + end) + else + pymethod((_o, _n=pynone) -> pythrow(pyiounsupportedoperation("truncate"))) + end + t["seek"] = if hasmethod(seek, Tuple{V, Int}) && hasmethod(position, Tuple{V}) + pymethod((_o, offset, whence=pyint(0)) -> begin + o = pyjlgetvalue(_o, V) + n = pyconvert(Int, offset) + w = pyconvert(Int, whence) + if w == 0 + seek(o, n) + elseif w == 1 + seek(o, position(o)+n) + elseif w == 2 + seekend(o) + seek(o, position(o)+n) + else + pythrow(pyvalueerror("Unsupported whence: $w")) + end + pyint(position(o)) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("seek"))) + end + t["__iter__"] = pymethod(o -> o) + t["__next__"] = pymethod(o -> (x=o.readline(); pylen(x)==0 ? pythrow(pystopiteration()) : x)) +end + +### RawIO (as RawIOBase) + +abstract type RawIO{V<:IO} <: PyJlSubclass{V} end + +pyjlrawio(o::IO) = pyjl(o, RawIO{typeof(o)}) + +pyjl_abc(::Type{T}) where {T<:RawIO} = pyiomodule.RawIOBase + +### BufferedIO (as BufferedIOBase) + +abstract type BufferedIO{V<:IO} <: PyJlSubclass{V} end + +pyjl_abc(::Type{T}) where {T<:BufferedIO} = pyiomodule.BufferedIOBase + +pyjlbufferedio(o::IO) = pyjl(o, BufferedIO{typeof(o)}) + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:BufferedIO, V} + t["detach"] = pymethod(o -> pythrow(pyiounsupportedoperation("detach"))) + t["readinto"] = if hasmethod(readbytes!, Tuple{V, Vector{UInt8}}) + pymethod((_o, _b) -> begin + b = PyBuffer(_b, C.PyBUF_WRITABLE) + o = pyjlgetvalue(_o, V) + n = readbytes!(o, unsafe_wrap(Array{UInt8}, Ptr{UInt8}(b.buf), b.len)) + pyint(n) + end) + else + pymethod((o, b) -> pythrow(pyiounsupportedoperation("readinto"))) + end + t["read"] = if hasmethod(read, Tuple{V}) && hasmethod(read, Tuple{V, Int}) + pymethod((_o, size=pynone) -> begin + n = pyconvert(Union{Int,Nothing}, size) + o = pyjlgetvalue(_o, V) + pybytes(convert(Vector{UInt8}, (n===nothing || n < 0) ? read(o) : read(o, n))) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("read"))) + end + t["write"] = if hasmethod(write, Tuple{V, Vector{UInt8}}) + pymethod((_o, _b) -> begin + b = PyBuffer(_b, C.PyBUF_SIMPLE) + o = pyjlgetvalue(_o, V) + n = write(o, unsafe_wrap(Array{UInt8}, Ptr{UInt8}(b.buf), b.len)) + pyint(n) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("write"))) + end + t["readline"] = if hasmethod(read, Tuple{V, Type{UInt8}}) + pymethod((_o, size=pynone) -> begin + n = pyconvert(Union{Int,Nothing}, size) + o = pyjlgetvalue(_o, V) + data = UInt8[] + while !eof(o) && (n===nothing || n < 0 || length(data) ≤ n) + c = read(o, UInt8) + push!(data, c) + c == 0x0A && break + end + pybytes(data) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("readline"))) + end +end + +### TextIO (as TextIOBase) + +abstract type TextIO{V<:IO} <: PyJlSubclass{V} end + +pyjl_abc(::Type{T}) where {T<:TextIO} = pyiomodule.TextIOBase + +pyjltextio(o::IO) = pyjl(o, TextIO{typeof(o)}) + +function pyjl_addattrs(t, ::Type{T}, ::Type{V}) where {T<:TextIO, V} + t["encoding"] = pyproperty(o -> pystr("utf-8")) + t["errors"] = pyproperty(o -> pystr("strict")) + t["newlines"] = pyproperty(o -> pynone) + t["detach"] = pymethod(o -> pythrow(pyiounsupportedoperation("detach"))) + t["read"] = if hasmethod(read, Tuple{V, Type{Char}}) + pymethod((_o, size=pynone) -> begin + n = pytryconvert(Union{Nothing,Int}, size) + n === PyConvertFail() && pythrow(pytypeerror("'size' must be 'int' or 'None'")) + o = pyjlgetvalue(_o, V) + b = IOBuffer() + i = 0 + while !eof(o) && (n===nothing || n < 0 || i < n) + i += 1 + write(b, read(o, Char)) + end + seekstart(b) + pystr(read(b, String)) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("read"))) + end + t["readline"] = if hasmethod(read, Tuple{V, Type{Char}}) + pymethod((_o, size=pynone) -> begin + n = pytryconvert(Union{Nothing,Int}, size) + n === PyConvertFail() && pythrow(pytypeerror("'size' must be 'int' or 'None'")) + o = pyjlgetvalue(_o, V) + b = IOBuffer() + i = 0 + while !eof(o) && (n === nothing || n < 0 || i < n) + i += 1 + c = read(o, Char) + if c == '\n' + write(b, '\n') + break + elseif c == '\r' + write(b, '\n') + !eof(o) && peek(o, Char)=='\n' && read(o, Char) + break + else + write(b, c) + end + end + seekstart(b) + pystr(read(b, String)) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("readline"))) + end + t["write"] = if hasmethod(write, Tuple{V, String}) + pymethod((_o, _x) -> begin + x = pystr_asjuliastring(_x) + o = pyjlgetvalue(_o) + n = 0 + linesep = pystr_asjuliastring(pyosmodule.linesep) + i = firstindex(x) + while true + j = findnext('\n', x, i) + if j === nothing + y = SubString(x, i, lastindex(x)) + write(o, y) + n += length(y) + break + else + y = SubString(x, i, prevind(x, j)) + write(o, y) + write(o, linesep) + n += length(y) + 1 + i = nextind(x, j) + end + end + pyint(n) + end) + else + pymethod((args...) -> pythrow(pyiounsupportedoperation("write"))) + end +end + +### Buffer Protocol + +isflagset(flags, mask) = (flags & mask) == mask + +const PYJLBUFCACHE = Dict{Ptr{Cvoid}, Any}() + +function pyjl_get_buffer_impl(o, buf, flags, ptr, elsz, len, ndim, fmt, sz, strds, mutable) + b = UnsafePtr(buf) + c = [] + + # not influenced by flags: obj, buf, len, itemsize, ndim + b.obj[] = C_NULL + b.buf[] = ptr + b.itemsize[] = elsz + b.len[] = elsz * len + b.ndim[] = ndim + + # readonly + if isflagset(flags, C.PyBUF_WRITABLE) + if mutable + b.readonly[] = 1 + else + pythrow(pybuffererror("not writable")) + end + else + b.readonly[] = mutable ? 0 : 1 + end + + # format + if isflagset(flags, C.PyBUF_FORMAT) + b.format[] = cacheptr!(c, fmt) + else + b.format[] = C_NULL + end + + # shape + if isflagset(flags, C.PyBUF_ND) + b.shape[] = cacheptr!(c, C.Py_ssize_t[sz...]) + else + b.shape[] = C_NULL + end + + # strides + if isflagset(flags, C.PyBUF_STRIDES) + b.strides[] = cacheptr!(c, C.Py_ssize_t[(strds .* elsz)...]) + else + if size_to_cstrides(1, sz...) != strds + pythrow(pybuffererror("not C contiguous and strides not requested")) + end + b.strides[] = C_NULL + end + + # check contiguity + if isflagset(flags, C.PyBUF_C_CONTIGUOUS) + if size_to_cstrides(1, sz...) != strds + pythrow(pybuffererror("not C contiguous")) + end + end + if isflagset(flags, C.PyBUF_F_CONTIGUOUS) + if size_to_fstrides(1, sz...) != strds + pythrow(pybuffererror("not Fortran contiguous")) + end + end + if isflagset(flags, C.PyBUF_ANY_CONTIGUOUS) + if size_to_cstrides(1, sz...) != strds && size_to_fstrides(1, sz...) != strds + pythrow(pybuffererror("not contiguous")) + end + end + + # suboffsets + b.suboffsets[] = C_NULL + + # internal + cptr = Base.pointer_from_objref(c) + PYJLBUFCACHE[cptr] = c + b.internal[] = cptr + + # obj + b.obj[] = pyptr(pyincref!(o)) + Cint(0) +end + +pyjl_get_buffer(o, buf, flags) = pyjl_get_buffer(o, buf, flags, pyjlgetvalue(o)) +pyjl_get_buffer(o, buf, flags, x::AbstractArray) = + pyjl_get_buffer_impl(o, buf, flags, pointer(x), Base.aligned_sizeof(eltype(x)), length(x), ndims(x), pybufferformat(eltype(x)), size(x), strides(x), ismutablearray(x)) +pyjl_get_buffer(o, buf, flags, x::PyObjectArray) = + pyjl_get_buffer_impl(o, buf, flags, pointer(x.ptrs), sizeof(CPyPtr), length(x), ndims(x), "O", size(x), strides(x.ptrs), true) diff --git a/src/pandas.jl b/src/pandas.jl deleted file mode 100644 index 0657f7b4..00000000 --- a/src/pandas.jl +++ /dev/null @@ -1,110 +0,0 @@ -const pypandas = pylazyobject(() -> pyimport("pandas")) -const pypandasdataframetype = pylazyobject(() -> pypandas.DataFrame) - -asvector(x::AbstractVector) = x -asvector(x) = collect(x) - -""" - pycolumntable(src) - -Construct a "column table" from the `Tables.jl`-compatible table `src`, namely a Python `dict` mapping column names to column vectors. -""" -function pycolumntable(src) - cols = Tables.columns(src) - pydict_fromiter(pystr(String(n)) => asvector(Tables.getcolumn(cols, n)) for n in Tables.columnnames(cols)) -end -pycolumntable(; cols...) = pycolumntable(cols) -export pycolumntable - -""" - pyrowtable(src) - -Construct a "row table" from the `Tables.jl`-compatible table `src`, namely a Python `list` of rows, each row being a Python `dict` mapping column names to values. -""" -function pyrowtable(src) - rows = Tables.rows(src) - names = Tables.columnnames(rows) - pynames = [pystr(String(n)) for n in names] - pylist_fromiter(pydict_fromiter(pn => Tables.getcolumn(row, n) for (n,pn) in zip(names, pynames)) for row in rows) -end -pyrowtable(; cols...) = pyrowtable(cols) -export pyrowtable - -""" - pypandasdataframe([src]; ...) - -Construct a pandas dataframe from `src`. - -Usually equivalent to `pyimport("pandas").DataFrame(src, ...)`, but `src` may also be `Tables.jl`-compatible table. -""" -pypandasdataframe(t::PyObject; opts...) = pypandasdataframetype(t; opts...) -pypandasdataframe(; opts...) = pypandasdataframetype(; opts...) -function pypandasdataframe(t; opts...) - if Tables.istable(t) - cs = Tables.columns(t) - pypandasdataframetype(pydict_fromstringiter(string(c) => asvector(Tables.getcolumn(cs, c)) for c in Tables.columnnames(cs)); opts...) - else - pypandasdataframetype(t; opts...) - end -end -export pypandasdataframe - -multidict(src) = Dict(k=>v for (ks,v) in src for k in (ks isa Vector ? ks : [ks])) - -""" - PyPandasDataFrame(o; indexname=:index, columntypes=(), copy=false) - -Wrap the Pandas dataframe `o` as a Julia table. - -This object satisfies the `Tables.jl` and `TableTraits.jl` interfaces. - -- `:indexname` is the name of the index column when converting this to a table, and may be `nothing` to exclude the index. -- `:columntypes` is an iterable of `columnname=>type` or `[columnnames...]=>type` pairs, used when converting to a table. -- `:copy` is true to copy columns on conversion. -""" -struct PyPandasDataFrame - o :: PyObject - indexname :: Union{Symbol, Nothing} - columntypes :: Dict{Symbol, Type} - copy :: Bool -end -PyPandasDataFrame(o::PyObject; indexname=:index, columntypes=(), copy=false) = PyPandasDataFrame(o, indexname, multidict(columntypes), copy) -export PyPandasDataFrame - -function pypandasdataframe_tryconvert(::Type{T}, o::PyObject) where {T} - if PyPandasDataFrame <: T - return PyPandasDataFrame(o) - else - tryconvert(T, PyPandasDataFrame(o)) - end -end - -Base.show(io::IO, x::PyPandasDataFrame) = print(io, x.o) - -### Tables.jl / TableTraits.jl integration - -Tables.istable(::Type{PyPandasDataFrame}) = true -Tables.columnaccess(::Type{PyPandasDataFrame}) = true -function Tables.columns(x::PyPandasDataFrame) - # columns - names = Symbol[Symbol(pystr(String, c)) for c in x.o.columns] - columns = PyVector[PyVector{get(x.columntypes, c, missing)}(x.o[pystr(c)].to_numpy()) for c in names] - # index - if x.indexname !== nothing - if x.indexname ∈ names - error("table already has column called $(x.indexname), cannot use it for index") - else - pushfirst!(names, x.indexname) - pushfirst!(columns, PyArray{get(x.columntypes, x.indexname, missing)}(x.o.index.to_numpy())) - end - end - if x.copy - columns = map(copy, columns) - end - return NamedTuple{Tuple(names)}(Tuple(columns)) -end - -IteratorInterfaceExtensions.isiterable(x::PyPandasDataFrame) = true -IteratorInterfaceExtensions.getiterator(x::PyPandasDataFrame) = IteratorInterfaceExtensions.getiterator(Tables.rows(x)) - -TableTraits.isiterabletable(x::PyPandasDataFrame) = true diff --git a/src/pywith.jl b/src/pywith.jl deleted file mode 100644 index ec62d326..00000000 --- a/src/pywith.jl +++ /dev/null @@ -1,28 +0,0 @@ -""" - pywith(f, o) - -Simulates the Python `with` statement, calling `f(x)` where `x=o.__enter__()`. -""" -function pywith(f, o::PyObject) - enter = pytype(o).__enter__ - exit = pytype(o).__exit__ - value = enter(o) - hit_except = false - try - return f(value) - catch err - if err isa PythonRuntimeError - hit_except = true - if pytruth(exit(o, err.t, err.v, err.b)) - rethrow() - end - else - rethrow() - end - finally - if !hit_except - exit(o, pynone, pynone, pynone) - end - end -end -export pywith diff --git a/src/range.jl b/src/range.jl deleted file mode 100644 index cc6ca11f..00000000 --- a/src/range.jl +++ /dev/null @@ -1,64 +0,0 @@ -const pyrangetype = pylazyobject(() -> pybuiltins.range) -export pyrangetype - -pyrange(args...; opts...) = pyrangetype(args...; opts...) -pyrange(x::AbstractRange{<:Integer}) = pyrange(first(x), last(x)+sign(step(x)), step(x)) -export pyrange - -pyisrange(o::PyObject) = pytypecheck(o, pyrangetype) -export pyisrange - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {A<:Integer, C<:Integer, T<:StepRange{A,C}} - a = pyint_tryconvert(A, o.start) - a === PyConvertFail() && return a - b = pyint_tryconvert(A, o.stop) - b === PyConvertFail() && return b - c = pyint_tryconvert(C, o.step) - c === PyConvertFail() && return c - return StepRange{A,C}(a, c, b-oftype(b, sign(c))) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {A<:Integer, T<:UnitRange{A}} - o.step == pyint(1) || return PyConvertFail() - a = pyint_tryconvert(A, o.start) - a === PyConvertFail() && return a - b = pyint_tryconvert(A, o.stop) - b === PyConvertFail() && return b - return UnitRange{A}(a, b-one(b)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:StepRange} - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:UnitRange} - tryconvert(T, pyrange_tryconvert(UnitRange{BigInt}, o)) -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {I<:Integer, T<:AbstractRange{I}} - if (S = _typeintersect(T, StepRange{I,I})) != Union{} - pyrange_tryconvert(S, o) - elseif (S = _typeintersect(T, UnitRange{I})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{I, I}, o)) - end -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T<:AbstractRange{<:Integer}} - if (S = _typeintersect(T, StepRange{<:Integer, <:Integer})) != Union{} - pyrange_tryconvert(S, o) - elseif (S = _typeintersect(T, UnitRange{<:Integer})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) - end -end - -function pyrange_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, AbstractRange{<:Integer})) != Union{} - pyrange_tryconvert(S, o) - else - tryconvert(T, pyrange_tryconvert(StepRange{BigInt, BigInt}, o)) - end -end diff --git a/src/sequence.jl b/src/sequence.jl deleted file mode 100644 index 53b6c537..00000000 --- a/src/sequence.jl +++ /dev/null @@ -1,2 +0,0 @@ -pycontains(o, v) = check(Bool, C.PySequence_Contains(pyobject(o), pyobject(v))) -export pycontains diff --git a/src/set.jl b/src/set.jl deleted file mode 100644 index 863e0083..00000000 --- a/src/set.jl +++ /dev/null @@ -1,14 +0,0 @@ -const pysettype = pylazyobject(() -> pybuiltins.set) -const pyfrozensettype = pylazyobject(() -> pybuiltins.frozenset) -export pysettype, pyfrozensettype - -pyisset(o::PyObject) = pytypecheck(o, pysettype) -pyisfrozenset(o::PyObject) = pytypecheck(o, pyfrozensettype) -pyisanyset(o::PyObject) = pyisset(o) || pyisfrozenset(o) -export pyisset, pyisfrozenset, pyisanyset - -pyset(args...; opts...) = pysettype(args...; opts...) -pyset(x::Union{Tuple,AbstractVector}) = pyset(pylist(x)) -pyfrozenset(args...; opts...) = pyfrozensettype(args...; opts...) -pyfrozenset(x::Union{Tuple,AbstractVector}) = pyfrozenset(pylist(x)) -export pyset, pyfrozenset diff --git a/src/slice.jl b/src/slice.jl deleted file mode 100644 index 196a4840..00000000 --- a/src/slice.jl +++ /dev/null @@ -1,8 +0,0 @@ -const pyslicetype = pylazyobject(() -> pybuiltins.slice) -export pyslicetype - -pyslice(args...; opts...) = pyslicetype(args...; opts...) -export pyslice - -pyisslice(o::PyObject) = pytypecheck(o, pyslicetype) -export pyisslice diff --git a/src/stdlib.jl b/src/stdlib.jl deleted file mode 100644 index 3fbf01ef..00000000 --- a/src/stdlib.jl +++ /dev/null @@ -1,13 +0,0 @@ -for m in ["os", "sys", "pprint", "traceback", "numbers", "math", "collections", "collections.abc", "datetime", "fractions", "io", "types", "pdb"] - j = Symbol(:py, replace(m, '.'=>""), :module) - @eval const $j = pylazyobject(() -> pyimport($m)) -end - -""" - pynewclass(name, bases=(), kwargs=nothing; attrs...) - -Create a new Python class with the given name, base classes and attributes. -""" -pynewclass(name, bases=(), kwds=nothing; attrs...) = - pytypesmodule.new_class(name, bases, kwds, pyjlfunction(ns -> (for (k,v) in attrs; ns[string(k)] = v; end))) -export pynewclass diff --git a/src/str.jl b/src/str.jl deleted file mode 100644 index f42bbaeb..00000000 --- a/src/str.jl +++ /dev/null @@ -1,33 +0,0 @@ -const pystrtype = pylazyobject(() -> pybuiltins.str) -export pystrtype - -pystr(x::Union{String,SubString{String}}) = check(C.PyUnicode_DecodeUTF8(x, ncodeunits(x), C_NULL)) -pystr(x::AbstractString) = pystr(convert(String, x)) -pystr(x::AbstractChar) = pystr(String(x)) -pystr(x::Symbol) = pystr(String(x)) -pystr(args...; opts...) = pystrtype(args...; opts...) -export pystr - -pystr_asutf8string(o::PyObject) = check(C.PyUnicode_AsUTF8String(o)) - -pystr_asjuliastring(o::PyObject) = pybytes_asjuliastring(pystr_asutf8string(o)) - -pyisstr(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_UNICODE_SUBCLASS) -export pyisstr - -function pystr_tryconvert(::Type{T}, o::PyObject) where {T} - x = pystr_asjuliastring(o) - if (S = _typeintersect(T, String)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, AbstractString)) != Union{} - convert(S, x) - elseif (S = _typeintersect(T, Symbol)) != Union{} - convert(S, Symbol(x)) - elseif (S = _typeintersect(T, Char)) != Union{} - length(x)==1 ? convert(S, x[1]) : PyConvertFail() - elseif (S = _typeintersect(T, AbstractChar)) != Union{} - length(x)==1 ? convert(S, x[1]) : PyConvertFail() - else - tryconvert(T, x) - end -end diff --git a/src/tuple.jl b/src/tuple.jl deleted file mode 100644 index 54ef6b44..00000000 --- a/src/tuple.jl +++ /dev/null @@ -1,28 +0,0 @@ -const pytupletype = pylazyobject(() -> pybuiltins.tuple) -export pytupletype - -pyistuple(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_TUPLE_SUBCLASS) -export pyistuple - -pytuple() = check(C.PyTuple_New(0)) -pytuple(args...; opts...) = pytupletype(args...; opts...) -pytuple(x::Union{Tuple,AbstractVector,Pair}) = pytuple_fromiter(x) -export pytuple - -function pytuple_fromiter(xs) - n = length(xs) - t = check(C.PyTuple_New(n)) - for (i,x) in enumerate(xs) - xo = pyobject(x) - check(C.PyTuple_SetItem(t, i-1, pyincref!(xo))) - end - return t -end - -function pytuple_tryconvert(::Type{T}, o::PyObject) where {T} - if (S = _typeintersect(T, Tuple)) != Union{} - pyiterable_tryconvert(S, o) - else - PyConvertFail() - end -end diff --git a/src/type.jl b/src/type.jl deleted file mode 100644 index 9496242e..00000000 --- a/src/type.jl +++ /dev/null @@ -1,12 +0,0 @@ -const pytypetype = pylazyobject(() -> pybuiltins.type) -export pytypetype - -pyistype(o::PyObject) = pytypecheckfast(o, C.Py_TPFLAGS_TYPE_SUBCLASS) -export pyistype - -pytype(o::PyObject) = pyborrowedobject(C.Py_Type(o)) -export pytype - -pytypecheck(o::PyObject, t::PyObject) = !iszero(C.Py_TypeCheck(o, t)) - -pytypecheckfast(o::PyObject, f::Integer) = !iszero(C.Py_TypeCheckFast(o, f)) diff --git a/src/utils.jl b/src/utils.jl index 435ca3e8..e300b032 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,6 +28,7 @@ pybufferformat(::Type{T}) where {T} = T == Complex{Float64} ? "=Zd" : T == Bool ? "?" : T == Ptr{Cvoid} ? "P" : + T == C.PyObjectRef ? "O" : if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) n = fieldcount(T) flds = [] @@ -60,24 +61,129 @@ pybufferformat_to_type(fmt::AbstractString) = fmt == "d" ? Cdouble : fmt == "?" ? Bool : fmt == "P" ? Ptr{Cvoid} : - fmt == "O" ? CPyObjRef : + fmt == "O" ? C.PyObjectRef : fmt == "=e" ? Float16 : fmt == "=f" ? Float32 : fmt == "=d" ? Float64 : error("not implemented: $(repr(fmt))") -### TYPE UTILITIES +islittleendian() = Base.ENDIAN_BOM == 0x04030201 ? true : Base.ENDIAN_BOM == 0x01020304 ? false : error() -struct PyConvertFail end +pytypestrdescr(::Type{T}) where {T} = begin + c = islittleendian() ? '<' : '>' + T == Bool ? ("$(c)b$(sizeof(Bool))", nothing) : + T == Int8 ? ("$(c)i1", nothing) : + T == UInt8 ? ("$(c)u1", nothing) : + T == Int16 ? ("$(c)i2", nothing) : + T == UInt16 ? ("$(c)u2", nothing) : + T == Int32 ? ("$(c)i4", nothing) : + T == UInt32 ? ("$(c)u4", nothing) : + T == Int64 ? ("$(c)i8", nothing) : + T == UInt64 ? ("$(c)u8", nothing) : + T == Float16 ? ("$(c)f2", nothing) : + T == Float32 ? ("$(c)f4", nothing) : + T == Float64 ? ("$(c)f8", nothing) : + T == Complex{Float16} ? ("$(c)c4", nothing) : + T == Complex{Float32} ? ("$(c)c8", nothing) : + T == Complex{Float64} ? ("$(c)c16", nothing) : + T == C.PyObjectRef ? ("|O", nothing) : + if isstructtype(T) && isconcretetype(T) && Base.allocatedinline(T) + n = fieldcount(T) + flds = [] + for i in 1:n + nm = fieldname(T, i) + tp = fieldtype(T, i) + ts, ds = pytypestrdescr(tp) + isempty(ts) && return ("", nothing) + push!(flds, (nm isa Integer ? "f$(nm-1)" : string(nm), ds === nothing ? ts : ds)) + d = (i==n ? sizeof(T) : fieldoffset(T, i+1)) - (fieldoffset(T, i) + sizeof(tp)) + @assert d≥0 + d>0 && push!(flds, ("", "|V$(d)")) + end + ("|$(sizeof(T))V", flds) + else + ("", nothing) + end +end -tryconvert(::Type{T}, x) where {T} = -try - convert(T, x) -catch - PyConvertFail() +pytypestrdescr_to_type(ts::String, descr) = begin + # byte swapped? + bsc = ts[1] + bs = bsc=='<' ? !islittleendian() : bsc=='>' ? islittleendian() : bsc=='|' ? false : error("endianness character not supported: $ts") + bs && error("byte-swapping not supported: $ts") + # element type + etc = ts[2] + if etc == 'b' + sz = parse(Int, ts[3:end]) + sz == sizeof(Bool) && return Bool + error("bool of this size not supported: $ts") + elseif etc == 'i' + sz = parse(Int, ts[3:end]) + sz == 1 && return Int8 + sz == 2 && return Int16 + sz == 4 && return Int32 + sz == 8 && return Int64 + sz == 16 && return Int128 + error("signed int of this size not supported: $ts") + elseif etc == 'u' + sz = parse(Int, ts[3:end]) + sz == 1 && return UInt8 + sz == 2 && return UInt16 + sz == 4 && return UInt32 + sz == 8 && return UInt64 + sz == 16 && return UInt128 + error("unsigned int of this size not supported: $ts") + elseif etc == 'f' + sz = parse(Int, ts[3:end]) + sz == 2 && return Float16 + sz == 4 && return Float32 + sz == 8 && return Float64 + error("float of this size not supported: $ts") + elseif etc == 'c' + sz = parse(Int, ts[3:end]) + sz == 4 && return Complex{Float16} + sz == 8 && return Complex{Float32} + sz == 16 && return Complex{Float64} + error("complex of this size not supported: $ts") + elseif etc == 'O' + return C.PyObjectRef + else + error("type not supported: $ts") + end end + +### TYPE UTILITIES + +# Used to signal a Python error from functions that return general Julia objects +struct PYERR end + +# Used to signal that conversion failed +struct NOTIMPLEMENTED end + +# Somewhere to stash results +const RESULT = Ref{Any}(nothing) +putresult(x) = (RESULT[] = x; 1) +putresult(x::PYERR) = -1 +putresult(x::NOTIMPLEMENTED) = 0 +takeresult(::Type{T}=Any) where {T} = (r = RESULT[]::T; RESULT[] = nothing; r) + +tryconvert(::Type{T}, x::PYERR) where {T} = PYERR() +tryconvert(::Type{T}, x::NOTIMPLEMENTED) where {T} = NOTIMPLEMENTED() tryconvert(::Type{T}, x::T) where {T} = x -tryconvert(::Type{T}, x::PyConvertFail) where {T} = x +tryconvert(::Type{T}, x) where {T} = + try + convert(T, x) + catch + NOTIMPLEMENTED() + end + +CTryConvertRule_wrapref(o, ::Type{S}) where {S} = putresult(S(C.PyObjectRef(o))) +CTryConvertRule_trywrapref(o, ::Type{S}) where {S} = + try + putresult(S(C.PyObjectRef(o))) + catch + 0 + end @generated _typeintersect(::Type{T1}, ::Type{T2}) where {T1,T2} = typeintersect(T1, T2) @@ -101,3 +207,10 @@ function pointer_from_obj(o::T) where {T} end p, c end + +isnull(p::Ptr) = Ptr{Cvoid}(p) == C_NULL +isnull(p::UnsafePtr) = isnull(pointer(p)) +ism1(x::T) where {T<:Number} = x == (zero(T)-one(T)) + +# Put something in here to keep it around forever +const CACHE = []