Skip to content
This repository was archived by the owner on Jul 1, 2023. It is now read-only.

Commit 91b6449

Browse files
compnerdcompnerd
authored
build: bump to TensorFlow 2.4.0 (#1156)
- Remove the 2.3 compatibility path - Update the install rules to install additional headers for X10 - Update the documentation to point to a 2.4 build of TensorFlow - Update the include paths for the internal use of TensorFlow - Update macOS/Windows CI rules to use the 2.4 release build
1 parent 843c51b commit 91b6449

File tree

6 files changed

+47
-87
lines changed

6 files changed

+47
-87
lines changed

‎.github/workflows/macOS.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ jobs:
2222
2323
- name: Install X10
2424
run: |
25-
curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip
25+
curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip
2626
unzip tensorflow-darwin-x64.zip
27-
mv tensorflow-darwin-x64/Library/tensorflow-2.3.0 ~/Library/
27+
mv tensorflow-darwin-x64/Library/tensorflow-2.4.0 ~/Library/
2828
2929
- name: Build
3030
run: |
31-
TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib
31+
TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib
3232
# - name: Run tests
3333
# run: |
34-
# DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.3.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib
34+
# DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.4.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib
3535

‎.github/workflows/windows.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,16 @@ jobs:
3232
3333
- name: Install X10
3434
run: |
35-
Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip
35+
Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip
3636
Expand-Archive -Force -Path tensorflow-windows-x64.zip -DestinationPath C:\Library\
37-
Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.3.0 C:\Library
38-
echo "C:\Library\tensorflow-2.3.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
37+
Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.4.0 C:\Library
38+
echo "C:\Library\tensorflow-2.4.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
3939
4040
- name: Build
4141
run: |
4242
# We build twice as the first one currently fails for some unexplained
4343
# reason. This should only build once.
44-
swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib
45-
swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib
44+
swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib
45+
swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib
4646
# - name: Run tests
4747
# run: swift test -v

‎CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
167167
GIT_REPOSITORY
168168
git://github.com/tensorflow/tensorflow
169169
GIT_TAG
170-
r2.3
170+
r2.4
171171
UPDATE_DISCONNECTED
172172
TRUE
173173
CONFIGURE_COMMAND
@@ -207,6 +207,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
207207
set(X10_LIBRARY ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX})
208208
set(X10_INCLUDE_DIRS
209209
${SOURCE_DIR}
210+
${SOURCE_DIR}/tensorflow
210211
${SOURCE_DIR}/bazel-bin
211212
${SOURCE_DIR}/bazel-libtensorflow/external/com_google_absl
212213
${SOURCE_DIR}/bazel-libtensorflow/external/com_google_protobuf/src

‎Documentation/Development.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -409,8 +409,8 @@ $ swift test -Xcc -I/Library/tensorflow-2.4.0/usr/include -Xlinker -L/Library/te
409409

410410
[swift]: https://swift.org/download/#snapshots
411411
[cmake]: https://www.cmake.org/download
412-
[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip
413-
[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip
412+
[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip
413+
[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip
414414
[bazel]: https://docs.bazel.build/versions/master/install.html
415415
[bazelisk]: https://github.com/bazelbuild/bazelisk
416416
[configure.py]: https://github.com/tensorflow/tensorflow/blob/master/configure.py

‎Sources/CTensorFlow/CMakeLists.txt‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,21 @@ list(GET X10_INCLUDE_DIRS 0 X10_SOURCE_DIR)
1212

1313
install(FILES include/module.modulemap include/CTensorFlow.h
1414
DESTINATION lib/swift/tensorflow)
15-
install(DIRECTORY ${X10_SOURCE_DIR}/tensorflow/c
16-
DESTINATION lib/swift/tensorflow
17-
FILES_MATCHING
18-
PATTERN c_api.h
19-
PATTERN c_api_experimental.h
20-
PATTERN tf_attrtype.h
21-
PATTERN tf_datatype.h
22-
PATTERN tf_status.h
23-
PATTERN tf_tensor.h
24-
PATTERN eager/c_api.h)
15+
install(FILES
16+
${X10_SOURCE_DIR}/tensorflow/c/c_api.h
17+
${X10_SOURCE_DIR}/tensorflow/c/c_api_experimental.h
18+
${X10_SOURCE_DIR}/tensorflow/c/c_api_macros.h
19+
${X10_SOURCE_DIR}/tensorflow/c/tf_attrtype.h
20+
${X10_SOURCE_DIR}/tensorflow/c/tf_datatype.h
21+
${X10_SOURCE_DIR}/tensorflow/c/tf_file_statistics.h
22+
${X10_SOURCE_DIR}/tensorflow/c/tf_status.h
23+
${X10_SOURCE_DIR}/tensorflow/c/tf_tensor.h
24+
${X10_SOURCE_DIR}/tensorflow/c/tf_tstring.h
25+
DESTINATION lib/swift/tensorflow/c)
26+
install(FILES
27+
${X10_SOURCE_DIR}/tensorflow/c/eager/c_api.h
28+
DESTINATION lib/swift/tensorflow/c/eager)
29+
install(FILES
30+
${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring.h
31+
${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring_internal.h
32+
DESTINATION lib/swift/tensorflow/core/platform)

‎Sources/TensorFlow/Core/StringTensor.swift‎

Lines changed: 16 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -47,72 +47,23 @@ extension StringTensor {
4747
// utf8CString is null-terminated. TF APIs want the strings without null-terminators.
4848
let cStrings = scalars.map { $0.utf8CString.dropLast() }
4949

50-
// Note: `TENSORFLOW_MASTER` changes below are necessary for the new TensorFlow ABI-stable
51-
// unified string tensor design.
52-
#if TENSORFLOW_MASTER
53-
let byteCount = scalars.count * MemoryLayout<TF_TString>.stride
54-
55-
let handle = TensorHandle<String>(
56-
shape: shape.dimensions,
57-
byteCount: byteCount,
58-
bufferInitializer: { tensorBuffer in
59-
var dataAddr =
60-
tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count)
61-
for cString in cStrings {
62-
TF_TString_Init(dataAddr)
63-
cString.withUnsafeBufferPointer { buffer in
64-
TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count)
65-
}
66-
dataAddr = dataAddr.advanced(by: 1)
67-
}
68-
})
69-
self.init(handle: handle)
70-
#else
71-
let tfEncodedSizes = cStrings.map { TF_StringEncodedSize($0.count) }
72-
73-
// Format information copied from tensorflow/c/c_api.h:
74-
// The format for TF_STRING tensors is:
75-
// start_offset: array[uint64]
76-
// data: byte[...]
77-
//
78-
// The string length (as a varint), followed by the contents of the string is encoded at
79-
// data[start_offset[i]]].
80-
// The size of the "start_offset" region.
81-
let startOffsetsByteCount = scalars.count * MemoryLayout<UInt64>.stride
82-
83-
// The size of the "data" region.
84-
let dataByteCount = tfEncodedSizes.reduce(0, +) * MemoryLayout<UInt8>.stride
85-
86-
let handle = TensorHandle<String>(
87-
shape: shape.dimensions,
88-
byteCount: startOffsetsByteCount + dataByteCount,
89-
bufferInitializer: { tensorBuffer in
90-
// Initialize the "start_offset" region.
91-
var startOffset: UInt64 = 0
92-
var startOffsetAddr =
93-
tensorBuffer.bindMemory(to: UInt64.self, capacity: scalars.count)
94-
for tfEncodedSize in tfEncodedSizes {
95-
startOffsetAddr.initialize(to: startOffset)
96-
startOffsetAddr = startOffsetAddr.advanced(by: 1)
97-
startOffset = startOffset + UInt64(tfEncodedSize)
98-
}
99-
100-
// Initialize the "data" region.
101-
var dataAddr = tensorBuffer.advanced(by: startOffsetsByteCount)
102-
.bindMemory(to: Int8.self, capacity: dataByteCount)
103-
let status = TF_NewStatus()
104-
for (cString, tfEncodedSize) in zip(cStrings, tfEncodedSizes) {
105-
_ = cString.withUnsafeBufferPointer { buffer in
106-
TF_StringEncode(
107-
buffer.baseAddress, buffer.count, dataAddr, tfEncodedSize, status)
108-
}
109-
checkOk(status)
110-
dataAddr = dataAddr.advanced(by: tfEncodedSize)
50+
let byteCount = scalars.count * MemoryLayout<TF_TString>.stride
51+
52+
let handle = TensorHandle<String>(
53+
shape: shape.dimensions,
54+
byteCount: byteCount,
55+
bufferInitializer: { tensorBuffer in
56+
var dataAddr =
57+
tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count)
58+
for cString in cStrings {
59+
TF_TString_Init(dataAddr)
60+
cString.withUnsafeBufferPointer { buffer in
61+
TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count)
11162
}
112-
TF_DeleteStatus(status)
113-
})
114-
self.init(handle: handle)
115-
#endif
63+
dataAddr = dataAddr.advanced(by: 1)
64+
}
65+
})
66+
self.init(handle: handle)
11667
}
11768

11869
/// Creates a 0-D `StringTensor` from a scalar value.

0 commit comments

Comments
 (0)