Skip to content

[WIP] remove expensive api from InternalRow #6869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[WIP] remove expensive api from InternalRow #6869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 2 additions & 104 deletions sql/catalyst/src/main/java/org/apache/spark/sql/BaseRow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@

package org.apache.spark.sql;

import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.List;

import scala.collection.Seq;
import scala.collection.mutable.ArraySeq;

Expand All @@ -36,17 +31,6 @@ final public int length() {
return size();
}

@Override
public boolean anyNull() {
final int n = size();
for (int i=0; i < n; i++) {
if (isNullAt(i)) {
return true;
}
}
return false;
}

@Override
public StructType schema() { throw new UnsupportedOperationException(); }

Expand Down Expand Up @@ -90,78 +74,13 @@ public boolean getBoolean(int i) {
throw new UnsupportedOperationException();
}

@Override
public String getString(int i) {
throw new UnsupportedOperationException();
}

@Override
public BigDecimal getDecimal(int i) {
throw new UnsupportedOperationException();
}

@Override
public Date getDate(int i) {
throw new UnsupportedOperationException();
}

@Override
public Timestamp getTimestamp(int i) {
throw new UnsupportedOperationException();
}

@Override
public <T> Seq<T> getSeq(int i) {
throw new UnsupportedOperationException();
}

@Override
public <T> List<T> getList(int i) {
throw new UnsupportedOperationException();
}

@Override
public <K, V> scala.collection.Map<K, V> getMap(int i) {
throw new UnsupportedOperationException();
}

@Override
public <T> scala.collection.immutable.Map<String, T> getValuesMap(Seq<String> fieldNames) {
throw new UnsupportedOperationException();
}

@Override
public <K, V> java.util.Map<K, V> getJavaMap(int i) {
throw new UnsupportedOperationException();
}

@Override
public Row getStruct(int i) {
throw new UnsupportedOperationException();
}

@Override
public <T> T getAs(int i) {
throw new UnsupportedOperationException();
}

@Override
public <T> T getAs(String fieldName) {
throw new UnsupportedOperationException();
}

@Override
public int fieldIndex(String name) {
throw new UnsupportedOperationException();
}

/**
* A generic version of Row.equals(Row), which is used for tests.
*/
@Override
public boolean equals(Object other) {
if (other instanceof Row) {
Row row = (Row) other;
if (other instanceof InternalRow) {
InternalRow row = (InternalRow) other;
int n = size();
if (n != row.size()) {
return false;
Expand All @@ -186,7 +105,6 @@ public InternalRow copy() {
return new GenericRow(arr);
}

@Override
public Seq<Object> toSeq() {
final int n = size();
final ArraySeq<Object> values = new ArraySeq<Object>(n);
Expand All @@ -195,24 +113,4 @@ public Seq<Object> toSeq() {
}
return values;
}

@Override
public String toString() {
return mkString("[", ",", "]");
}

@Override
public String mkString() {
return toSeq().mkString();
}

@Override
public String mkString(String sep) {
return toSeq().mkString(sep);
}

@Override
public String mkString(String start, String sep, String end) {
return toSeq().mkString(start, sep, end);
}
}
99 changes: 18 additions & 81 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@

package org.apache.spark.sql

import scala.util.hashing.MurmurHash3

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericRow
import org.apache.spark.sql.types.StructType

Expand Down Expand Up @@ -118,17 +117,17 @@ object Row {
*
* @group row
*/
trait Row extends Serializable {
trait Row extends InternalRow {
/** Number of elements in the Row. */
def size: Int = length
override def size: Int = length

/** Number of elements in the Row. */
def length: Int
override def length: Int

/**
* Schema for the row.
*/
def schema: StructType = null
override def schema: StructType = null

/**
* Returns the value at position i. If the value is null, null is returned. The following
Expand All @@ -153,7 +152,7 @@ trait Row extends Serializable {
* StructType -> org.apache.spark.sql.Row
* }}}
*/
def apply(i: Int): Any
override def apply(i: Int): Any
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why we override it without implement it?


/**
* Returns the value at position i. If the value is null, null is returned. The following
Expand All @@ -178,50 +177,50 @@ trait Row extends Serializable {
* StructType -> org.apache.spark.sql.Row
* }}}
*/
def get(i: Int): Any = apply(i)
override def get(i: Int): Any = apply(i)

/** Checks whether the value at position i is null. */
def isNullAt(i: Int): Boolean
override def isNullAt(i: Int): Boolean

/**
* Returns the value at position i as a primitive boolean.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getBoolean(i: Int): Boolean
override def getBoolean(i: Int): Boolean

/**
* Returns the value at position i as a primitive byte.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getByte(i: Int): Byte
override def getByte(i: Int): Byte

/**
* Returns the value at position i as a primitive short.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getShort(i: Int): Short
override def getShort(i: Int): Short

/**
* Returns the value at position i as a primitive int.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getInt(i: Int): Int
override def getInt(i: Int): Int

/**
* Returns the value at position i as a primitive long.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getLong(i: Int): Long
override def getLong(i: Int): Long

/**
* Returns the value at position i as a primitive float.
Expand All @@ -230,23 +229,23 @@ trait Row extends Serializable {
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getFloat(i: Int): Float
override def getFloat(i: Int): Float

/**
* Returns the value at position i as a primitive double.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getDouble(i: Int): Double
override def getDouble(i: Int): Double

/**
* Returns the value at position i as a String object.
*
* @throws ClassCastException when data type does not match.
* @throws NullPointerException when value is null.
*/
def getString(i: Int): String
override def getString(i: Int): String

/**
* Returns the value at position i of decimal type as java.math.BigDecimal.
Expand Down Expand Up @@ -313,7 +312,7 @@ trait Row extends Serializable {
*
* @throws ClassCastException when data type does not match.
*/
def getAs[T](i: Int): T = apply(i).asInstanceOf[T]
override def getAs[T](i: Int): T = apply(i).asInstanceOf[T]

/**
* Returns the value of a given fieldName.
Expand Down Expand Up @@ -347,70 +346,8 @@ trait Row extends Serializable {
}.toMap
}

override def toString(): String = s"[${this.mkString(",")}]"

/**
* Make a copy of the current [[Row]] object.
*/
def copy(): Row

/** Returns true if there are any NULL values in this row. */
def anyNull: Boolean = {
val len = length
var i = 0
while (i < len) {
if (isNullAt(i)) { return true }
i += 1
}
false
}

override def equals(that: Any): Boolean = that match {
case null => false
case that: Row =>
if (this.length != that.length) {
return false
}
var i = 0
val len = this.length
while (i < len) {
if (apply(i) != that.apply(i)) {
return false
}
i += 1
}
true
case _ => false
}

override def hashCode: Int = {
// Using Scala's Seq hash code implementation.
var n = 0
var h = MurmurHash3.seqSeed
val len = length
while (n < len) {
h = MurmurHash3.mix(h, apply(n).##)
n += 1
}
MurmurHash3.finalizeHash(h, n)
}

/* ---------------------- utility methods for Scala ---------------------- */

/**
* Return a Scala Seq representing the row. ELements are placed in the same order in the Seq.
*/
def toSeq: Seq[Any]

/** Displays all elements of this sequence in a string (without a separator). */
def mkString: String = toSeq.mkString

/** Displays all elements of this sequence in a string using a separator string. */
def mkString(sep: String): String = toSeq.mkString(sep)

/**
* Displays all elements of this traversable or iterator in a string using
* start, end, and separator strings.
*/
def mkString(start: String, sep: String, end: String): String = toSeq.mkString(start, sep, end)
override def copy(): Row
}
Loading