Switch to LiteRT repo (#3199)

* Switch from the github.com/tensorflow/tensorflow repository to the new github.com/google-ai-edge/LiteRT repository for TFLite sources.

* Fix up include paths.

* More fixes.

* Fix more paths.

* Re-add missing python fixes.

* Re-add missing python fixes.

* Re-run `bazel run //third_party:python_requirements.update -- --upgrade` with Python 3.12.

* Revert third_party/python_requirements.txt.

---------

Co-authored-by: Esun Kim <[email protected]>

Author: gonnet

ci/sync_from_upstream_tf.sh

@@ -27,20 +27,33 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR=${SCRIPT_DIR}/..
 cd "${ROOT_DIR}"
 
-rm -rf /tmp/tensorflow
+rm -rf /tmp/litert
 
-git clone https://github.com/tensorflow/tensorflow.git --depth=1 /tmp/tensorflow
+git clone https://github.com/google-ai-edge/LiteRT.git --depth=1 /tmp/litert
 
-SHARED_TFL_CODE=$(<ci/tflite_files.txt)
+SHARED_TFL_CODE=$(grep -v tflite/converter ci/tflite_files.txt)
+SHARED_CONVERTER_CODE=$(grep tflite/converter ci/tflite_files.txt)
 
 # Delete all the shared TFL/TFLM code prior to copying from upstream to ensure
 # no stale files are left in the tree.
 rm -f $(find tensorflow/lite/ -type d \( -path tensorflow/lite/experimental -o -path tensorflow/lite/micro \) -prune -false -o -name "*.cc" -o -name "*.c" -o -name "*.h" -o -name "*.py" -o -name "*.fbs")
 
+# Copy all the right files from the TFLite repo to all the right places in this repo.
 for filepath in ${SHARED_TFL_CODE}
 do
-  mkdir -p $(dirname ${filepath})
-  /bin/cp /tmp/tensorflow/${filepath} ${filepath}
+  local_filepath=${filepath//tflite\//tensorflow\/lite\/}
+  mkdir -p $(dirname ${local_filepath})
+  /bin/cp /tmp/litert/${filepath} ${local_filepath}
+  sed -i 's/tflite\/converter\//tensorflow\/compiler\/mlir\/lite\//' ${local_filepath}
+  sed -i 's/tflite\//tensorflow\/lite\//' ${local_filepath}
+done
+for filepath in ${SHARED_CONVERTER_CODE}
+do
+  local_filepath=${filepath//tflite\/converter\//tensorflow\/compiler\/mlir\/lite\/}
+  mkdir -p $(dirname ${local_filepath})
+  /bin/cp /tmp/litert/${filepath} ${local_filepath}
+  sed -i 's/tflite\/converter\//tensorflow\/compiler\/mlir\/lite\//' ${local_filepath}
+  sed -i 's/tflite\//tensorflow\/lite\//' ${local_filepath}
 done
 
 # https://github.com/tensorflow/tflite-micro/pull/8
@@ -68,7 +81,7 @@ bazel clean
 # repositories (TF and tflite-micro) which needs the import statements to be
 # modified.
 PY_FILES=$(find tensorflow/lite/tools tensorflow/lite/python -name "*.py")
-sed -i 's/from tensorflow\.lite/from tflite_micro\.tensorflow\.lite/' ${PY_FILES}
+sed -i 's/from tflite/from tflite_micro\.tensorflow\.lite/' ${PY_FILES}
 
 # Since the TFLM code was deleted from the tensorflow repository, the
 # microfrontend is no longer sync'd from upstream and instead maintaned as a

ci/tflite_files.txt

@@ -1,123 +1,123 @@
-tensorflow/compiler/mlir/lite/core/api/error_reporter.h
-tensorflow/compiler/mlir/lite/core/api/error_reporter.cc
-tensorflow/compiler/mlir/lite/core/c/builtin_op_data.h
-tensorflow/compiler/mlir/lite/core/c/tflite_types.h
-tensorflow/compiler/mlir/lite/schema/schema.fbs
-tensorflow/compiler/mlir/lite/schema/schema_utils.h
-tensorflow/compiler/mlir/lite/schema/schema_utils.cc
-tensorflow/lite/array.h
-tensorflow/lite/array.cc
-tensorflow/lite/builtin_op_data.h
-tensorflow/lite/builtin_ops.h
-tensorflow/lite/context_util.h
-tensorflow/lite/core/api/flatbuffer_conversions.cc
-tensorflow/lite/core/api/tensor_utils.cc
-tensorflow/lite/core/c/common.cc
-tensorflow/lite/kernels/internal/common.cc
-tensorflow/lite/kernels/internal/quantization_util.cc
-tensorflow/lite/kernels/internal/tensor_ctypes.cc
-tensorflow/lite/kernels/internal/reference/comparisons.cc
-tensorflow/lite/kernels/kernel_util.cc
-tensorflow/lite/c/builtin_op_data.h
-tensorflow/lite/c/c_api_types.h
-tensorflow/lite/c/common.h
-tensorflow/lite/core/api/error_reporter.h
-tensorflow/lite/core/api/flatbuffer_conversions.h
-tensorflow/lite/core/api/tensor_utils.h
-tensorflow/lite/core/c/builtin_op_data.h
-tensorflow/lite/core/c/c_api_types.h
-tensorflow/lite/core/c/common.h
-tensorflow/lite/core/macros.h
-tensorflow/lite/kernels/internal/common.h
-tensorflow/lite/kernels/internal/compatibility.h
-tensorflow/lite/kernels/internal/portable_tensor_utils.h
-tensorflow/lite/kernels/internal/portable_tensor_utils.cc
-tensorflow/lite/kernels/internal/quantization_util.h
-tensorflow/lite/kernels/internal/reference/add.h
-tensorflow/lite/kernels/internal/reference/add_n.h
-tensorflow/lite/kernels/internal/reference/arg_min_max.h
-tensorflow/lite/kernels/internal/reference/batch_to_space_nd.h
-tensorflow/lite/kernels/internal/reference/batch_matmul.h
-tensorflow/lite/kernels/internal/reference/binary_function.h
-tensorflow/lite/kernels/internal/reference/broadcast_args.h
-tensorflow/lite/kernels/internal/reference/broadcast_to.h
-tensorflow/lite/kernels/internal/reference/ceil.h
-tensorflow/lite/kernels/internal/reference/comparisons.h
-tensorflow/lite/kernels/internal/reference/concatenation.h
-tensorflow/lite/kernels/internal/reference/conv.h
-tensorflow/lite/kernels/internal/reference/cumsum.h
-tensorflow/lite/kernels/internal/reference/depth_to_space.h
-tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h
-tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
-tensorflow/lite/kernels/internal/reference/dequantize.h
-tensorflow/lite/kernels/internal/reference/div.h
-tensorflow/lite/kernels/internal/reference/elu.h
-tensorflow/lite/kernels/internal/reference/exp.h
-tensorflow/lite/kernels/internal/reference/fill.h
-tensorflow/lite/kernels/internal/reference/floor.h
-tensorflow/lite/kernels/internal/reference/floor_div.h
-tensorflow/lite/kernels/internal/reference/floor_mod.h
-tensorflow/lite/kernels/internal/reference/fully_connected.h
-tensorflow/lite/kernels/internal/reference/hard_swish.h
-tensorflow/lite/kernels/internal/reference/integer_ops/add.h
-tensorflow/lite/kernels/internal/reference/integer_ops/conv.h
-tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h
-tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h
-tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h
-tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h
-tensorflow/lite/kernels/internal/reference/integer_ops/mean.h
-tensorflow/lite/kernels/internal/reference/integer_ops/mul.h
-tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h
-tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h
-tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h
-tensorflow/lite/kernels/internal/reference/l2normalization.h
-tensorflow/lite/kernels/internal/reference/leaky_relu.h
-tensorflow/lite/kernels/internal/reference/log_softmax.h
-tensorflow/lite/kernels/internal/reference/maximum_minimum.h
-tensorflow/lite/kernels/internal/reference/mul.h
-tensorflow/lite/kernels/internal/reference/neg.h
-tensorflow/lite/kernels/internal/reference/pad.h
-tensorflow/lite/kernels/internal/reference/pooling.h
-tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
-tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
-tensorflow/lite/kernels/internal/reference/portable_tensor_utils_impl.h
-tensorflow/lite/kernels/internal/reference/prelu.h
-tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h
-tensorflow/lite/kernels/internal/reference/quantize.h
-tensorflow/lite/kernels/internal/reference/reduce.h
-tensorflow/lite/kernels/internal/reference/requantize.h
-tensorflow/lite/kernels/internal/reference/resize_bilinear.h
-tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h
-tensorflow/lite/kernels/internal/reference/reverse.h
-tensorflow/lite/kernels/internal/reference/round.h
-tensorflow/lite/kernels/internal/reference/softmax.h
-tensorflow/lite/kernels/internal/reference/space_to_batch_nd.h
-tensorflow/lite/kernels/internal/reference/space_to_depth.h
-tensorflow/lite/kernels/internal/reference/sub.h
-tensorflow/lite/kernels/internal/reference/logistic.h
-tensorflow/lite/kernels/internal/reference/lstm_cell.h
-tensorflow/lite/kernels/internal/reference/select.h
-tensorflow/lite/kernels/internal/reference/slice.h
-tensorflow/lite/kernels/internal/reference/strided_slice.h
-tensorflow/lite/kernels/internal/reference/tanh.h
-tensorflow/lite/kernels/internal/reference/transpose.h
-tensorflow/lite/kernels/internal/reference/transpose_conv.h
-tensorflow/lite/kernels/internal/cppmath.h
-tensorflow/lite/kernels/internal/max.h
-tensorflow/lite/kernels/internal/min.h
-tensorflow/lite/kernels/internal/portable_tensor.h
-tensorflow/lite/kernels/internal/strided_slice_logic.h
-tensorflow/lite/kernels/internal/tensor_ctypes.h
-tensorflow/lite/kernels/internal/types.h
-tensorflow/lite/kernels/kernel_util.h
-tensorflow/lite/kernels/padding.h
-tensorflow/lite/portable_type_to_tflitetype.h
-tensorflow/lite/python/schema_util.py
-tensorflow/lite/schema/schema_utils.h
-tensorflow/lite/tools/flatbuffer_utils.py
-tensorflow/lite/tools/flatbuffer_utils_test.py
-tensorflow/lite/tools/randomize_weights.py
-tensorflow/lite/tools/strip_strings.py
-tensorflow/lite/tools/test_utils.py
-tensorflow/lite/tools/visualize.py
-tensorflow/lite/tools/visualize_test.py
+tflite/converter/core/api/error_reporter.h
+tflite/converter/core/api/error_reporter.cc
+tflite/converter/core/c/builtin_op_data.h
+tflite/converter/core/c/tflite_types.h
+tflite/converter/schema/schema.fbs
+tflite/converter/schema/schema_utils.h
+tflite/converter/schema/schema_utils.cc
+tflite/array.h
+tflite/array.cc
+tflite/builtin_op_data.h
+tflite/builtin_ops.h
+tflite/context_util.h
+tflite/core/api/flatbuffer_conversions.cc
+tflite/core/api/tensor_utils.cc
+tflite/core/c/common.cc
+tflite/kernels/internal/common.cc
+tflite/kernels/internal/quantization_util.cc
+tflite/kernels/internal/tensor_ctypes.cc
+tflite/kernels/internal/reference/comparisons.cc
+tflite/kernels/kernel_util.cc
+tflite/c/builtin_op_data.h
+tflite/c/c_api_types.h
+tflite/c/common.h
+tflite/core/api/error_reporter.h
+tflite/core/api/flatbuffer_conversions.h
+tflite/core/api/tensor_utils.h
+tflite/core/c/builtin_op_data.h
+tflite/core/c/c_api_types.h
+tflite/core/c/common.h
+tflite/core/macros.h
+tflite/kernels/internal/common.h
+tflite/kernels/internal/compatibility.h
+tflite/kernels/internal/portable_tensor_utils.h
+tflite/kernels/internal/portable_tensor_utils.cc
+tflite/kernels/internal/quantization_util.h
+tflite/kernels/internal/reference/add.h
+tflite/kernels/internal/reference/add_n.h
+tflite/kernels/internal/reference/arg_min_max.h
+tflite/kernels/internal/reference/batch_to_space_nd.h
+tflite/kernels/internal/reference/batch_matmul.h
+tflite/kernels/internal/reference/binary_function.h
+tflite/kernels/internal/reference/broadcast_args.h
+tflite/kernels/internal/reference/broadcast_to.h
+tflite/kernels/internal/reference/ceil.h
+tflite/kernels/internal/reference/comparisons.h
+tflite/kernels/internal/reference/concatenation.h
+tflite/kernels/internal/reference/conv.h
+tflite/kernels/internal/reference/cumsum.h
+tflite/kernels/internal/reference/depth_to_space.h
+tflite/kernels/internal/reference/depthwiseconv_float.h
+tflite/kernels/internal/reference/depthwiseconv_uint8.h
+tflite/kernels/internal/reference/dequantize.h
+tflite/kernels/internal/reference/div.h
+tflite/kernels/internal/reference/elu.h
+tflite/kernels/internal/reference/exp.h
+tflite/kernels/internal/reference/fill.h
+tflite/kernels/internal/reference/floor.h
+tflite/kernels/internal/reference/floor_div.h
+tflite/kernels/internal/reference/floor_mod.h
+tflite/kernels/internal/reference/fully_connected.h
+tflite/kernels/internal/reference/hard_swish.h
+tflite/kernels/internal/reference/integer_ops/add.h
+tflite/kernels/internal/reference/integer_ops/conv.h
+tflite/kernels/internal/reference/integer_ops/depthwise_conv.h
+tflite/kernels/internal/reference/integer_ops/fully_connected.h
+tflite/kernels/internal/reference/integer_ops/logistic.h
+tflite/kernels/internal/reference/integer_ops/l2normalization.h
+tflite/kernels/internal/reference/integer_ops/mean.h
+tflite/kernels/internal/reference/integer_ops/mul.h
+tflite/kernels/internal/reference/integer_ops/pooling.h
+tflite/kernels/internal/reference/integer_ops/tanh.h
+tflite/kernels/internal/reference/integer_ops/transpose_conv.h
+tflite/kernels/internal/reference/l2normalization.h
+tflite/kernels/internal/reference/leaky_relu.h
+tflite/kernels/internal/reference/log_softmax.h
+tflite/kernels/internal/reference/maximum_minimum.h
+tflite/kernels/internal/reference/mul.h
+tflite/kernels/internal/reference/neg.h
+tflite/kernels/internal/reference/pad.h
+tflite/kernels/internal/reference/pooling.h
+tflite/kernels/internal/reference/portable_tensor_utils.cc
+tflite/kernels/internal/reference/portable_tensor_utils.h
+tflite/kernels/internal/reference/portable_tensor_utils_impl.h
+tflite/kernels/internal/reference/prelu.h
+tflite/kernels/internal/reference/process_broadcast_shapes.h
+tflite/kernels/internal/reference/quantize.h
+tflite/kernels/internal/reference/reduce.h
+tflite/kernels/internal/reference/requantize.h
+tflite/kernels/internal/reference/resize_bilinear.h
+tflite/kernels/internal/reference/resize_nearest_neighbor.h
+tflite/kernels/internal/reference/reverse.h
+tflite/kernels/internal/reference/round.h
+tflite/kernels/internal/reference/softmax.h
+tflite/kernels/internal/reference/space_to_batch_nd.h
+tflite/kernels/internal/reference/space_to_depth.h
+tflite/kernels/internal/reference/sub.h
+tflite/kernels/internal/reference/logistic.h
+tflite/kernels/internal/reference/lstm_cell.h
+tflite/kernels/internal/reference/select.h
+tflite/kernels/internal/reference/slice.h
+tflite/kernels/internal/reference/strided_slice.h
+tflite/kernels/internal/reference/tanh.h
+tflite/kernels/internal/reference/transpose.h
+tflite/kernels/internal/reference/transpose_conv.h
+tflite/kernels/internal/cppmath.h
+tflite/kernels/internal/max.h
+tflite/kernels/internal/min.h
+tflite/kernels/internal/portable_tensor.h
+tflite/kernels/internal/strided_slice_logic.h
+tflite/kernels/internal/tensor_ctypes.h
+tflite/kernels/internal/types.h
+tflite/kernels/kernel_util.h
+tflite/kernels/padding.h
+tflite/portable_type_to_tflitetype.h
+tflite/python/schema_util.py
+tflite/schema/schema_utils.h
+tflite/tools/flatbuffer_utils.py
+tflite/tools/flatbuffer_utils_test.py
+tflite/tools/randomize_weights.py
+tflite/tools/strip_strings.py
+tflite/tools/test_utils.py
+tflite/tools/visualize.py
+tflite/tools/visualize_test.py

tensorflow/compiler/mlir/lite/schema/schema.fbs

@@ -489,7 +489,7 @@ enum BuiltinOperator : int32 {
   STABLEHLO_CBRT = 208, // WARNING: No runtime support
   STABLEHLO_CASE = 209,
 }
-// LINT.ThenChange(nnapi_linter/linter.proto)
+// LINT.ThenChange(internal-1)
 
 // Options for the builtin operators.
 union BuiltinOptions {

tensorflow/lite/c/builtin_op_data.h

@@ -16,7 +16,7 @@ limitations under the License.
 #define TENSORFLOW_LITE_C_BUILTIN_OP_DATA_H_
 
 /// For documentation, see
-/// third_party/tensorflow/lite/core/c/builtin_op_data.h
+/// tensorflow/lite/core/c/builtin_op_data.h
 
 #include "tensorflow/lite/core/c/builtin_op_data.h"
 

tensorflow/lite/core/c/builtin_op_data.h

@@ -14,7 +14,7 @@ limitations under the License.
 ==============================================================================*/
 /// WARNING: Users of TensorFlow Lite should not include this file directly,
 /// but should instead include
-/// "third_party/tensorflow/lite/c/builtin_op_data.h".
+/// "tensorflow/lite/c/builtin_op_data.h".
 /// Only the TensorFlow Lite implementation itself should include this
 /// file directly.
 #ifndef TENSORFLOW_LITE_CORE_C_BUILTIN_OP_DATA_H_

tensorflow/lite/core/c/c_api_types.h

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 // WARNING: Users of TensorFlow Lite should not include this file directly, but
-// should instead include "third_party/tensorflow/lite/c/c_api_types.h".
+// should instead include "tensorflow/lite/c/c_api_types.h".
 // Only the TensorFlow Lite implementation itself should include this file
 // directly.
 
@@ -31,7 +31,7 @@ limitations under the License.
 // NOLINTEND(whitespace/line_length)
 // clang-format on
 
-// IWYU pragma: private, include "third_party/tensorflow/lite/c/c_api_types.h"
+// IWYU pragma: private, include "tensorflow/lite/c/c_api_types.h"
 
 #ifndef TENSORFLOW_LITE_CORE_C_C_API_TYPES_H_
 #define TENSORFLOW_LITE_CORE_C_C_API_TYPES_H_

tensorflow/lite/core/c/common.h

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 // WARNING: Users of TensorFlow Lite should not include this file directly, but
-// should instead include "third_party/tensorflow/lite/c/common.h".
+// should instead include "tensorflow/lite/c/common.h".
 // Only the TensorFlow Lite implementation itself should include this file
 // directly.
 
@@ -47,7 +47,7 @@ limitations under the License.
 // NOLINTEND(whitespace/line_length)
 // clang-format on
 
-// IWYU pragma: private, include "third_party/tensorflow/lite/c/common.h"
+// IWYU pragma: private, include "tensorflow/lite/c/common.h"
 
 #ifndef TENSORFLOW_LITE_CORE_C_COMMON_H_
 #define TENSORFLOW_LITE_CORE_C_COMMON_H_