Skip to content

Commit be73612

Browse files
saudetkarllessard
saudet
and
karllessard
authored
Upgrade for JavaCPP 1.5.7 and TensorFlow 2.8.0 (#415)
Co-authored-by: Karl Lessard <[email protected]>
1 parent 02ec490 commit be73612

File tree

151 files changed

+13562
-1054
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

151 files changed

+13562
-1054
lines changed

‎.github/workflows/ci.yml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ env:
1717
jobs:
1818
quick-build:
1919
if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI build')
20-
runs-on: ubuntu-latest
20+
runs-on: ubuntu-18.04
2121
container: centos:7
2222
steps:
2323
- name: Install environment
@@ -47,7 +47,7 @@ jobs:
4747
mvn compiler:compile -Pdev,jdk17 -B -U -e
4848
check-format:
4949
if: github.event_name == 'pull_request'
50-
runs-on: ubuntu-latest
50+
runs-on: ubuntu-18.04
5151
container: centos:7
5252
steps:
5353
- name: Install environment
@@ -76,7 +76,7 @@ jobs:
7676
run: |
7777
mvn spotless:check -Pdev,jdk17 -B -U -e
7878
prepare:
79-
runs-on: ubuntu-latest
79+
runs-on: ubuntu-18.04
8080
outputs:
8181
stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }}
8282
steps:
@@ -93,7 +93,7 @@ jobs:
9393
echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID"
9494
linux-x86_64:
9595
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
96-
runs-on: ubuntu-latest
96+
runs-on: ubuntu-18.04
9797
container: centos:7
9898
needs: prepare
9999
strategy:
@@ -113,7 +113,7 @@ jobs:
113113
tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/
114114
ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
115115
echo Downloading Bazel
116-
curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-linux-x86_64.sh -o bazel.sh --retry 10
116+
curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-linux-x86_64.sh -o bazel.sh --retry 10
117117
bash bazel.sh
118118
if [[ "${{ matrix.ext }}" == *-gpu ]]; then
119119
echo Installing CUDA
@@ -163,7 +163,7 @@ jobs:
163163
df -h
164164
macosx-x86_64:
165165
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
166-
runs-on: macos-latest
166+
runs-on: macos-10.15
167167
needs: prepare
168168
strategy:
169169
matrix:
@@ -173,7 +173,7 @@ jobs:
173173
run: |
174174
python3 -m pip install numpy six
175175
echo Downloading Bazel
176-
curl -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-installer-darwin-x86_64.sh -o bazel.sh --retry 10
176+
curl -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10
177177
bash bazel.sh
178178
brew install libomp perl
179179
- name: Configure Java
@@ -227,7 +227,7 @@ jobs:
227227
bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;"
228228
echo Downloading Bazel
229229
mkdir C:\bazel
230-
curl.exe -L https://github.com/bazelbuild/bazel/releases/download/3.7.2/bazel-3.7.2-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
230+
curl.exe -L https://github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
231231
set "EXT=${{ matrix.ext }}"
232232
if "%EXT:~-4%" == "-gpu" (
233233
echo Removing some unused stuff to avoid running out of disk space
@@ -287,7 +287,7 @@ jobs:
287287
deploy:
288288
if: github.event_name == 'push' && contains(github.ref, 'master')
289289
needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
290-
runs-on: ubuntu-latest
290+
runs-on: ubuntu-18.04
291291
steps:
292292
- name: Checkout repository
293293
uses: actions/checkout@v1

‎tensorflow-core/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
<javacpp.platform.macosx-x86_64.extension>macosx-x86_64${javacpp.platform.extension}</javacpp.platform.macosx-x86_64.extension>
6868
<javacpp.platform.windows-x86.extension>windows-x86${javacpp.platform.extension}</javacpp.platform.windows-x86.extension>
6969
<javacpp.platform.windows-x86_64.extension>windows-x86_64${javacpp.platform.extension}</javacpp.platform.windows-x86_64.extension>
70-
<javacpp.version>1.5.6</javacpp.version>
70+
<javacpp.version>1.5.7</javacpp.version>
7171
</properties>
7272

7373
<profiles>

‎tensorflow-core/tensorflow-core-api/.bazelrc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
build --remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm
2-
build --remote_upload_local_results=false
2+
build --remote_upload_local_results=false
3+
build --incompatible_restrict_string_escapes=false

‎tensorflow-core/tensorflow-core-api/.bazelversion

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.7.2
1+
4.2.1

‎tensorflow-core/tensorflow-core-api/WORKSPACE

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ http_archive(
1818
patch_args = ["-p1"],
1919
patch_cmds = ["grep -rl 'java_package' tensorflow/core | xargs sed -i.bak 's/^\(.* java_package = \"org\.tensorflow\.\)\(.*\"\)/\\1proto.\\2'/"],
2020
urls = [
21-
"https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.7.1.tar.gz",
21+
"https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.8.0.tar.gz",
2222
],
23-
sha256 = "abebe2cf5ca379e18071693ca5f45b88ade941b16258a21cc1f12d77d5387a21",
24-
strip_prefix = "tensorflow-2.7.1"
23+
sha256 = "66b953ae7fba61fd78969a2e24e350b26ec116cf2e6a7eb93d02c63939c6f9f7",
24+
strip_prefix = "tensorflow-2.8.0"
2525
)
2626

2727
# START: Upstream TensorFlow dependencies

‎tensorflow-core/tensorflow-core-api/build.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1
77

88
export BAZEL_VC="${VCINSTALLDIR:-}"
99
if [[ -d $BAZEL_VC ]]; then
10+
export BAZEL_BUILD="--output_user_root=$(cygpath -w $TMP) build"
1011
export BUILD_FLAGS="--copt=//arch:AVX `#--copt=//arch:AVX2` --define=override_eigen_strong_inline=true"
1112
export PYTHON_BIN_PATH=$(which python.exe)
1213
else
14+
export BAZEL_BUILD="build"
1315
export BUILD_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx `#--copt=-mavx2 --copt=-mfma` --linkopt=-lstdc++ --host_linkopt=-lstdc++"
1416
export PYTHON_BIN_PATH=$(which python3)
1517
fi
@@ -33,7 +35,7 @@ BUILD_FLAGS="$BUILD_FLAGS --experimental_repo_remote_exec --python_path="$PYTHON
3335
BUILD_FLAGS="$BUILD_FLAGS --distinct_host_configuration=true"
3436

3537
# Build C/C++ API of TensorFlow itself including a target to generate ops for Java
36-
bazel --bazelrc=tensorflow.bazelrc build $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \
38+
bazel --bazelrc=tensorflow.bazelrc $BAZEL_BUILD $BUILD_FLAGS ${BUILD_USER_FLAGS:-} \
3739
@org_tensorflow//tensorflow:tensorflow_cc \
3840
@org_tensorflow//tensorflow/tools/lib_package:jnilicenses_generate \
3941
:java_proto_gen_sources \

‎tensorflow-core/tensorflow-core-api/external/tensorflow-proto.patch

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,3 +461,25 @@ diff -ruN tensorflow-2.7.0/tensorflow/core/util/memmapped_file_system.proto tens
461461

462462
// A message that describes one region of memmapped file.
463463
message MemmappedFileSystemDirectoryElement {
464+
diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto
465+
--- tensorflow-2.8.0/tensorflow/core/protobuf/coordination_config.proto 2022-02-01 04:17:33.000000000 +0900
466+
+++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/coordination_config.proto 2022-02-16 11:02:10.162709816 +0900
467+
@@ -3,6 +3,7 @@
468+
package tensorflow;
469+
470+
option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto";
471+
+option java_package = "org.tensorflow.distruntime";
472+
473+
// Coordination service configuration parameters.
474+
// The system picks appropriate values for fields that are not set.
475+
diff -ruN tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto
476+
--- tensorflow-2.8.0/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-01 04:17:33.000000000 +0900
477+
+++ tensorflow-2.8.0-proto/tensorflow/core/protobuf/distributed_runtime_payloads.proto 2022-02-16 11:00:03.739373379 +0900
478+
@@ -4,6 +4,7 @@
479+
480+
option cc_enable_arenas = true;
481+
option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto";
482+
+option java_package = "org.tensorflow.distruntime";
483+
484+
// Used to serialize and transmit tensorflow::Status payloads through
485+
// grpc::Status `error_details` since grpc::Status lacks payload API.

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_AnonymousIteratorV2.pbtxt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
op {
22
graph_op_name: "AnonymousIteratorV2"
3-
visibility: VISIBLE
4-
endpoint {
5-
name: "data.AnonymousIterator"
6-
}
3+
visibility: SKIP
74
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_AnonymousIteratorV3.pbtxt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
op {
2+
graph_op_name: "AnonymousIteratorV3"
3+
visibility: VISIBLE
4+
endpoint {
5+
name: "data.AnonymousIterator"
6+
}
7+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_AnonymousMutableDenseHashTable.pbtxt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
op {
2+
graph_op_name: "AnonymousMutableDenseHashTable"
3+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_AnonymousMutableHashTable.pbtxt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
op {
2+
graph_op_name: "AnonymousMutableHashTable"
3+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_AnonymousMutableHashTableOfTensors.pbtxt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
op {
2+
graph_op_name: "AnonymousMutableHashTableOfTensors"
3+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_DataServiceDatasetV2.pbtxt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
op {
22
graph_op_name: "DataServiceDatasetV2"
3-
visibility: VISIBLE
4-
endpoint {
5-
name: "data.DataServiceDatasetV2"
6-
}
3+
visibility: SKIP
74
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_DataServiceDatasetV3.pbtxt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
op {
2+
graph_op_name: "DataServiceDatasetV3"
3+
visibility: VISIBLE
4+
endpoint {
5+
name: "data.DataServiceDataset"
6+
}
7+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_DynamicEnqueueTPUEmbeddingArbitraryTensorBatch.pbtxt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
op {
2+
graph_op_name: "DynamicEnqueueTPUEmbeddingArbitraryTensorBatch"
3+
endpoint {
4+
name: "tpu.DynamicEnqueueTPUEmbeddingArbitraryTensorBatch"
5+
}
6+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_EnqueueTPUEmbeddingArbitraryTensorBatch.pbtxt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
op {
2+
graph_op_name: "EnqueueTPUEmbeddingArbitraryTensorBatch"
3+
endpoint {
4+
name: "tpu.EnqueueTPUEmbeddingArbitraryTensorBatch"
5+
}
6+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_FileSystemSetConfiguration.pbtxt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
op {
2+
graph_op_name: "FileSystemSetConfiguration"
3+
}

‎tensorflow-core/tensorflow-core-api/src/bazel/api_def/import/api_import.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,8 @@ int main(int argc, char* argv[]) {
183183
TF_CHECK_OK(env->GetMatchingPaths(golden_api_path, &golden_api_files));
184184
LOG(INFO) << "Loading " << golden_api_files.size() << " Python API golden files";
185185
for (const auto& filename : golden_api_files) {
186-
// Skip the raw_ops API, as it contains all op endpoints
187-
if (filename == "tensorflow.raw_ops.pbtxt") {
186+
// Skip the raw_ops API, as it contains all op endpoints in a single package
187+
if (str_util::EndsWith(filename, "tensorflow.raw_ops.pbtxt")) {
188188
continue;
189189
}
190190
string contents;

‎tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/DataOps.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import org.tensorflow.op.data.ChooseFastestBranchDataset;
3333
import org.tensorflow.op.data.ChooseFastestDataset;
3434
import org.tensorflow.op.data.ConcatenateDataset;
35-
import org.tensorflow.op.data.DataServiceDatasetV2;
35+
import org.tensorflow.op.data.DataServiceDataset;
3636
import org.tensorflow.op.data.DatasetCardinality;
3737
import org.tensorflow.op.data.DatasetFromGraph;
3838
import org.tensorflow.op.data.DatasetToGraph;
@@ -350,16 +350,17 @@ public ConcatenateDataset concatenateDataset(Operand<? extends TType> inputDatas
350350
* @param iterationCounter The iterationCounter value
351351
* @param outputTypes The value of the outputTypes attribute
352352
* @param outputShapes The value of the outputShapes attribute
353+
* @param uncompressFn The value of the uncompressFn attribute
353354
* @param options carries optional attribute values
354-
* @return a new instance of DataServiceDatasetV2
355+
* @return a new instance of DataServiceDataset
355356
*/
356-
public DataServiceDatasetV2 dataServiceDatasetV2(Operand<TInt64> datasetId,
357+
public DataServiceDataset dataServiceDataset(Operand<TInt64> datasetId,
357358
Operand<TString> processingMode, Operand<TString> address, Operand<TString> protocol,
358359
Operand<TString> jobName, Operand<TInt64> consumerIndex, Operand<TInt64> numConsumers,
359360
Operand<TInt64> maxOutstandingRequests, Operand<? extends TType> iterationCounter,
360361
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes,
361-
DataServiceDatasetV2.Options... options) {
362-
return DataServiceDatasetV2.create(scope, datasetId, processingMode, address, protocol, jobName, consumerIndex, numConsumers, maxOutstandingRequests, iterationCounter, outputTypes, outputShapes, options);
362+
ConcreteFunction uncompressFn, DataServiceDataset.Options... options) {
363+
return DataServiceDataset.create(scope, datasetId, processingMode, address, protocol, jobName, consumerIndex, numConsumers, maxOutstandingRequests, iterationCounter, outputTypes, outputShapes, uncompressFn, options);
363364
}
364365

365366
/**
@@ -1594,12 +1595,14 @@ public SleepDataset sleepDataset(Operand<? extends TType> inputDataset,
15941595
* It must be positive.
15951596
* @param outputTypes The value of the outputTypes attribute
15961597
* @param outputShapes The value of the outputShapes attribute
1598+
* @param options carries optional attribute values
15971599
* @return a new instance of SlidingWindowDataset
15981600
*/
15991601
public SlidingWindowDataset slidingWindowDataset(Operand<? extends TType> inputDataset,
16001602
Operand<TInt64> windowSize, Operand<TInt64> windowShift, Operand<TInt64> windowStride,
1601-
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes) {
1602-
return SlidingWindowDataset.create(scope, inputDataset, windowSize, windowShift, windowStride, outputTypes, outputShapes);
1603+
List<Class<? extends TType>> outputTypes, List<Shape> outputShapes,
1604+
SlidingWindowDataset.Options... options) {
1605+
return SlidingWindowDataset.create(scope, inputDataset, windowSize, windowShift, windowStride, outputTypes, outputShapes, options);
16031606
}
16041607

16051608
/**

‎tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/IoOps.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,9 @@ public final class IoOps {
9090

9191
/**
9292
* Decode web-safe base64-encoded strings.
93-
* Input may or may not have padding at the end. See EncodeBase64 for padding.
94-
* Web-safe means that input must use - and _ instead of + and /.
93+
* Input may or may not have padding at the end. See
94+
* <a href="https://www.tensorflow.org/api_docs/python/tf/io/encode_base64">EncodeBase64</a>
95+
* for padding. Web-safe means that input must use - and _ instead of + and /.
9596
*
9697
* @param input Base64 strings to decode.
9798
* @return a new instance of DecodeBase64
@@ -242,8 +243,8 @@ public <T extends TType> DeserializeManySparse<T> deserializeManySparse(
242243

243244
/**
244245
* Encode strings into web-safe base64 format.
245-
* Refer to the following article for more information on base64 format:
246-
* en.wikipedia.org/wiki/Base64. Base64 strings may have padding with '=' at the
246+
* Refer to <a href="https://en.wikipedia.org/wiki/Base64">this article</a> for more information on
247+
* base64 format. Base64 strings may have padding with '=' at the
247248
* end so that the encoded has length multiple of 4. See Padding section of the
248249
* link above.
249250
* <p>Web-safe means that the encoder uses - and _ instead of + and /.
@@ -992,8 +993,8 @@ public WholeFileReader wholeFileReader(WholeFileReader.Options... options) {
992993
}
993994

994995
/**
995-
* Writes contents to the file at input filename. Creates file and recursively
996-
* creates directory if not existing.
996+
* Writes {@code contents} to the file at input {@code filename}.
997+
* Creates the file and recursively creates directory if it does not exist.
997998
*
998999
* @param filename scalar. The name of the file to which we write the contents.
9991000
* @param contents scalar. The content to be written to the output file.

0 commit comments

Comments
 (0)