Skip to content

Upgrade for JavaCPP 1.5.7 and TensorFlow 2.8.0 #415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Mar 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ env:
jobs:
quick-build:
if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI build')
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
container: centos:7
steps:
- name: Install environment
Expand Down Expand Up @@ -47,7 +47,7 @@ jobs:
mvn compiler:compile -Pdev,jdk17 -B -U -e
check-format:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
container: centos:7
steps:
- name: Install environment
Expand Down Expand Up @@ -76,7 +76,7 @@ jobs:
run: |
mvn spotless:check -Pdev,jdk17 -B -U -e
prepare:
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
outputs:
stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }}
steps:
Expand All @@ -93,7 +93,7 @@ jobs:
echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID"
linux-x86_64:
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
container: centos:7
needs: prepare
strategy:
Expand All @@ -113,7 +113,7 @@ jobs:
tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/
ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
echo Downloading Bazel
curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-linux-x86_64.sh -o bazel.sh --retry 10
curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-linux-x86_64.sh -o bazel.sh --retry 10
bash bazel.sh
if [[ "${{ matrix.ext }}" == *-gpu ]]; then
echo Installing CUDA
Expand Down Expand Up @@ -163,7 +163,7 @@ jobs:
df -h
macosx-x86_64:
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
runs-on: macos-latest
runs-on: macos-10.15
needs: prepare
strategy:
matrix:
Expand All @@ -173,7 +173,7 @@ jobs:
run: |
python3 -m pip install numpy six
echo Downloading Bazel
curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-darwin-x86_64.sh -o bazel.sh --retry 10
curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10
bash bazel.sh
brew install libomp perl
- name: Configure Java
Expand Down Expand Up @@ -227,7 +227,7 @@ jobs:
bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;"
echo Downloading Bazel
mkdir C:\bazel
curl.exe -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
curl.exe -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
set "EXT=${{ matrix.ext }}"
if "%EXT:~-4%" == "-gpu" (
echo Removing some unused stuff to avoid running out of disk space
Expand Down Expand Up @@ -287,7 +287,7 @@ jobs:
deploy:
if: github.event_name == 'push' && contains(github.ref, 'master')
needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
steps:
- name: Checkout repository
uses: actions/checkout@v1
Expand Down
2 changes: 1 addition & 1 deletion tensorflow-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
<javacpp.platform.macosx-x86_64.extension>macosx-x86_64${javacpp.platform.extension}</javacpp.platform.macosx-x86_64.extension>
<javacpp.platform.windows-x86.extension>windows-x86${javacpp.platform.extension}</javacpp.platform.windows-x86.extension>
<javacpp.platform.windows-x86_64.extension>windows-x86_64${javacpp.platform.extension}</javacpp.platform.windows-x86_64.extension>
<javacpp.version>1.5.6</javacpp.version>
<javacpp.version>1.5.7</javacpp.version>
</properties>

<profiles>
Expand Down
3 changes: 2 additions & 1 deletion tensorflow-core/tensorflow-core-api/.bazelrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
build --remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm
build --remote_upload_local_results=false
build --remote_upload_local_results=false
build --incompatible_restrict_string_escapes=false
2 changes: 1 addition & 1 deletion tensorflow-core/tensorflow-core-api/.bazelversion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.7.2
4.2.1
6 changes: 3 additions & 3 deletions tensorflow-core/tensorflow-core-api/WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ http_archive(
patch_args = ["-p1"],
patch_cmds = ["grep -rl 'java_package' tensorflow/core | xargs sed -i.bak 's/^\(.* java_package = \"org\.tensorflow\.\)\(.*\"\)/\\1proto.\\2'/"],
urls = [
"https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.7.1.tar.gz",
"https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.8.0.tar.gz",
],
sha256 = "abebe2cf5ca379e18071693ca5f45b88ade941b16258a21cc1f12d77d5387a21",
strip_prefix = "tensorflow-2.7.1"
sha256 = "66b953ae7fba61fd78969a2e24e350b26ec116cf2e6a7eb93d02c63939c6f9f7",
strip_prefix = "tensorflow-2.8.0"
)

# START: Upstream TensorFlow dependencies
Expand Down
4 changes: 3 additions & 1 deletion tensorflow-core/tensorflow-core-api/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1

export BAZEL_VC="${VCINSTALLDIR:-}"
if [[ -d $BAZEL_VC ]]; then
export BAZEL_BUILD="--output_user_root=$(cygpath -w $TMP) build"
export BUILD_FLAGS="--copt=//arch:AVX `#--copt=//arch:AVX2` --define=override_eigen_strong_inline=true"
export PYTHON_BIN_PATH=$(which python.exe)
else
export BAZEL_BUILD="build"
export BUILD_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx `#--copt=-mavx2 --copt=-mfma` --linkopt=-lstdc++ --host_linkopt=-lstdc++"
export PYTHON_BIN_PATH=$(which python3)
fi
Expand All @@ -33,7 +35,7 @@ BUILD_FLAGS="$BUILD_FLAGS --experimental_repo_remote_exec --python_path="$PYTHON
BUILD_FLAGS="$BUILD_FLAGS --distinct_host_configuration=true"

# Build C/C++ API of TensorFlow itself including a target to generate ops for Java
bazel --bazelrc=tensorflow.bazelrc build $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \
bazel --bazelrc=tensorflow.bazelrc $BAZEL_BUILD $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \
@org_tensorflow//tensorflow:tensorflow_cc \
@org_tensorflow//tensorflow/tools/lib_package:jnilicenses_generate \
:java_proto_gen_sources \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -461,3 +461,25 @@ diff -ruN tensorflow-2.7.0/tensorflow/core/util/memmapped_file_system.proto tens

// A message that describes one region of memmapped file.
message MemmappedFileSystemDirectoryElement {
diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto
--- tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto 2022-02-01 04:17:33.000000000 +0900
+++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto 2022-02-16 11:02:10.162709816 +0900
@@ -3,6 +3,7 @@
package tensorflow;

option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto";
+option java_package = "org.tensorflow.distruntime";

// Coordination service configuration parameters.
// The system picks appropriate values for fields that are not set.
diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto
--- tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-01 04:17:33.000000000 +0900
+++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-16 11:00:03.739373379 +0900
@@ -4,6 +4,7 @@

option cc_enable_arenas = true;
option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto";
+option java_package = "org.tensorflow.distruntime";

// Used to serialize and transmit tensorflow::Status payloads through
// grpc::Status `error_details` since grpc::Status lacks payload API.
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
op {
graph_op_name: "AnonymousIteratorV2"
visibility: VISIBLE
endpoint {
name: "data.AnonymousIterator"
}
visibility: SKIP
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
op {
graph_op_name: "AnonymousIteratorV3"
visibility: VISIBLE
endpoint {
name: "data.AnonymousIterator"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
op {
graph_op_name: "AnonymousMutableDenseHashTable"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
op {
graph_op_name: "AnonymousMutableHashTable"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
op {
graph_op_name: "AnonymousMutableHashTableOfTensors"
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
op {
graph_op_name: "DataServiceDatasetV2"
visibility: VISIBLE
endpoint {
name: "data.DataServiceDatasetV2"
}
visibility: SKIP
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
op {
graph_op_name: "DataServiceDatasetV3"
visibility: VISIBLE
endpoint {
name: "data.DataServiceDataset"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
op {
graph_op_name: "DynamicEnqueueTPUEmbeddingArbitraryTensorBatch"
endpoint {
name: "tpu.DynamicEnqueueTPUEmbeddingArbitraryTensorBatch"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
op {
graph_op_name: "EnqueueTPUEmbeddingArbitraryTensorBatch"
endpoint {
name: "tpu.EnqueueTPUEmbeddingArbitraryTensorBatch"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
op {
graph_op_name: "FileSystemSetConfiguration"
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ int main(int argc, char* argv[]) {
TF_CHECK_OK(env->GetMatchingPaths(golden_api_path, &golden_api_files));
LOG(INFO) << "Loading " << golden_api_files.size() << " Python API golden files";
for (const auto& filename : golden_api_files) {
// Skip the raw_ops API, as it contains all op endpoints
if (filename == "tensorflow.raw_ops.pbtxt") {
// Skip the raw_ops API, as it contains all op endpoints in a single package
if (str_util::EndsWith(filename, "tensorflow.raw_ops.pbtxt")) {
continue;
}
string contents;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.tensorflow.op.data.ChooseFastestBranchDataset;
import org.tensorflow.op.data.ChooseFastestDataset;
import org.tensorflow.op.data.ConcatenateDataset;
import org.tensorflow.op.data.DataServiceDatasetV2;
import org.tensorflow.op.data.DataServiceDataset;
import org.tensorflow.op.data.DatasetCardinality;
import org.tensorflow.op.data.DatasetFromGraph;
import org.tensorflow.op.data.DatasetToGraph;
Expand Down Expand Up @@ -350,16 +350,17 @@ public ConcatenateDataset concatenateDataset(Operand<? extends TType> inputDatas
* @param iterationCounter The iterationCounter value
* @param outputTypes The value of the outputTypes attribute
* @param outputShapes The value of the outputShapes attribute
* @param uncompressFn The value of the uncompressFn attribute
* @param options carries optional attribute values
* @return a new instance of DataServiceDatasetV2
* @return a new instance of DataServiceDataset
*/
public DataServiceDatasetV2 dataServiceDatasetV2(Operand<TInt64> datasetId,
public DataServiceDataset dataServiceDataset(Operand<TInt64> datasetId,
Operand<TString> processingMode, Operand<TString> address, Operand<TString> protocol,
Operand<TString> jobName, Operand<TInt64> consumerIndex, Operand<TInt64> numConsumers,
Operand<TInt64> maxOutstandingRequests, Operand<? extends TType> iterationCounter,
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes,
DataServiceDatasetV2.Options... options) {
return DataServiceDatasetV2.create(scope, datasetId, processingMode, address, protocol, jobName, consumerIndex, numConsumers, maxOutstandingRequests, iterationCounter, outputTypes, outputShapes, options);
ConcreteFunction uncompressFn, DataServiceDataset.Options... options) {
return DataServiceDataset.create(scope, datasetId, processingMode, address, protocol, jobName, consumerIndex, numConsumers, maxOutstandingRequests, iterationCounter, outputTypes, outputShapes, uncompressFn, options);
}

/**
Expand Down Expand Up @@ -1594,12 +1595,14 @@ public SleepDataset sleepDataset(Operand<? extends TType> inputDataset,
* It must be positive.
* @param outputTypes The value of the outputTypes attribute
* @param outputShapes The value of the outputShapes attribute
* @param options carries optional attribute values
* @return a new instance of SlidingWindowDataset
*/
public SlidingWindowDataset slidingWindowDataset(Operand<? extends TType> inputDataset,
Operand<TInt64> windowSize, Operand<TInt64> windowShift, Operand<TInt64> windowStride,
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes) {
return SlidingWindowDataset.create(scope, inputDataset, windowSize, windowShift, windowStride, outputTypes, outputShapes);
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes,
SlidingWindowDataset.Options... options) {
return SlidingWindowDataset.create(scope, inputDataset, windowSize, windowShift, windowStride, outputTypes, outputShapes, options);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@ public final class IoOps {

/**
* Decode web-safe base64-encoded strings.
* Input may or may not have padding at the end. See EncodeBase64 for padding.
* Web-safe means that input must use - and _ instead of + and /.
* Input may or may not have padding at the end. See
* <a href="https://www.tensorflow.org/api_docs/python/tf/io/encode_base64">EncodeBase64</a>
* for padding. Web-safe means that input must use - and _ instead of + and /.
*
* @param input Base64 strings to decode.
* @return a new instance of DecodeBase64
Expand Down Expand Up @@ -242,8 +243,8 @@ public <T extends TType> DeserializeManySparse<T> deserializeManySparse(

/**
* Encode strings into web-safe base64 format.
* Refer to the following article for more information on base64 format:
* en.wikipedia.org/wiki/Base64. Base64 strings may have padding with '=' at the
* Refer to <a href="https://en.wikipedia.org/wiki/Base64">this article</a> for more information on
* base64 format. Base64 strings may have padding with '=' at the
* end so that the encoded has length multiple of 4. See Padding section of the
* link above.
* <p>Web-safe means that the encoder uses - and _ instead of + and /.
Expand Down Expand Up @@ -992,8 +993,8 @@ public WholeFileReader wholeFileReader(WholeFileReader.Options... options) {
}

/**
* Writes contents to the file at input filename. Creates file and recursively
* creates directory if not existing.
* Writes {@code contents} to the file at input {@code filename}.
* Creates the file and recursively creates directory if it does not exist.
*
* @param filename scalar. The name of the file to which we write the contents.
* @param contents scalar. The content to be written to the output file.
Expand Down
Loading