Enable macOS M1 wheel builds

This commit contains fixes for `ml-metadata`'s build setup to enable wheel builds on macOS M1. The core build files have not been touched except for version increments of Bazel dependencies, and for `mariadbconnector-c`, whose build file needed to be updated to support the latest version. Several dependencies were upgraded to newer versions, with the update policy being "as recent as possible without breaking changes". A few gRPC dependencies were eliminated from `ml-metadata`'s workspace to avoid friction on subsequent gRPC updates (this includes e.g. `protobuf`, `zlib`). This change allows for M1 wheel builds without more external system dependencies outside of Bazel - the used build definitions contain M1 capabilities for sufficiently new versions of Bazel.
google · Oct 31, 2022 · 110418a · 110418a
1 parent 40743db
commit 110418a
Show file tree

Hide file tree

Showing 12 changed files with 105 additions and 119 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -16,13 +16,6 @@
 # be compiled upon c++17 or higher.
 build --cxxopt="-std=c++17"
 
-# Due to the invalid escape sequence in rules_foreign_cc and the bazel 4.0.0 updates
-# (https://github.com/bazelbuild/bazel/commit/73402fa4aa5b9de46c9a4042b75e6fb332ad4a7f).
-build --incompatible_restrict_string_escapes=false
-
-# icu@: In create_linking_context: in call to create_linking_context(),
-# parameter 'user_link_flags' is deprecated and will be removed soon.
-# It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false
-build --incompatible_require_linker_input_cc_api=false
+# Support for Apple M1.
 build:macos --apple_platform_type=macos
 build:macos_arm64 --cpu=darwin_arm64
diff --git a/WORKSPACE b/WORKSPACE
@@ -4,48 +4,73 @@ load("//ml_metadata:repo.bzl", "mlmd_http_archive", "clean_dep")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-# lts_20210324.2
+# lts_20220623.1
+ABSL_COMMIT = "8c0b94e793a66495e0b1f34a5eb26bd7dc672db0"
 http_archive(
     name = "com_google_absl",
-    sha256 = "1764491a199eb9325b177126547f03d244f86b4ff28f16f206c7b3e7e4f777ec",
-    strip_prefix = "abseil-cpp-278e0a071885a22dcd2fd1b5576cc44757299343",
+    sha256 = "a6be76f59c474a215f2df5116b312257462e97f2e38b2bfa6df8b6a55710b058",
+    strip_prefix = "abseil-cpp-%s" % ABSL_COMMIT,
     urls = [
-        "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/278e0a071885a22dcd2fd1b5576cc44757299343.tar.gz",
-        "https://github.com/abseil/abseil-cpp/archive/278e0a071885a22dcd2fd1b5576cc44757299343.tar.gz"
+        "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/%s.tar.gz" % ABSL_COMMIT,
+        "https://github.com/abseil/abseil-cpp/archive/%s.tar.gz" % ABSL_COMMIT,
     ],
 )
 
 # rules_cc defines rules for generating C++ code from Protocol Buffers.
+RULES_CC_COMMIT = "dc039dc2d82696ab3f0087b6dfe79776d1ab2326" # 19/10/2022
 http_archive(
     name = "rules_cc",
-    sha256 = "35f2fb4ea0b3e61ad64a369de284e4fbbdcdba71836a5555abb5e194cf119509",
-    strip_prefix = "rules_cc-624b5d59dfb45672d4239422fa1e3de1822ee110",
+    sha256 = "391811a18486dbc745627fdaeea9a6d96b683ab88bbecb9fe880a29d4da8cb36",
+    strip_prefix = "rules_cc-%s" % RULES_CC_COMMIT,
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/624b5d59dfb45672d4239422fa1e3de1822ee110.tar.gz",
-        "https://github.com/bazelbuild/rules_cc/archive/624b5d59dfb45672d4239422fa1e3de1822ee110.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/%s.tar.gz" % RULES_CC_COMMIT,
+        "https://github.com/bazelbuild/rules_cc/archive/%s.tar.gz" % RULES_CC_COMMIT,
     ],
 )
 
+RULES_FOREIGN_CC_COMMIT = "a7f9e8b38cc2c8a7c66862dd6a4c8848e9829a02" # 23/09/2022
+http_archive(
+    name="rules_foreign_cc",
+    sha256 = "ce3121834693d76308f50fa62c548c2710f900d807beb11d97c25739b6995f58",
+    strip_prefix = "rules_foreign_cc-%s" % RULES_FOREIGN_CC_COMMIT,
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_foreign_cc/archive/%s.tar.gz" % RULES_FOREIGN_CC_COMMIT,
+        "https://github.com/bazelbuild/rules_foreign_cc/archive/%s.tar.gz" % RULES_FOREIGN_CC_COMMIT,
+    ],
+)
+
+PLATFORMS_COMMIT = "d9296d956de6e7b3ae83ecc37a5970e5eebeee32"
+http_archive(
+    name="platforms",
+    sha256 = "01db13196adfb8468db77d7e42b1828dd7242112cca55b4df65df7cf7c91ec39",
+    strip_prefix = "platforms-%s" % PLATFORMS_COMMIT,
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/platforms/archive/%s.tar.gz" % PLATFORMS_COMMIT,
+        "https://github.com/bazelbuild/PLATFORMS/archive/%s.tar.gz" % PLATFORMS_COMMIT,
+    ],
+)
+
+BORINGSSL_COMMIT = "adde128bd706c6caf26dd595e631871b09f40bf6"
 mlmd_http_archive(
     name = "boringssl",
-    sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45",
-    strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514",
+    sha256 = "7fc7632ce455575025be2cac610a23377753b01a4932d94a452cf831a5b32e9e",
+    strip_prefix = "boringssl-%s" % BORINGSSL_COMMIT,
     system_build_file = clean_dep("//ml_metadata/third_party/systemlibs:boringssl.BUILD"),
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz",
-        "https://github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/boringssl/archive/%s.tar.gz" % BORINGSSL_COMMIT,
+        "https://github.com/google/boringssl/archive/%s.tar.gz" % BORINGSSL_COMMIT,
     ],
 )
 
 mlmd_http_archive(
     name = "org_sqlite",
     build_file = clean_dep("//ml_metadata/third_party:sqlite.BUILD"),
-    sha256 = "adf051d4c10781ea5cfabbbc4a2577b6ceca68590d23b58b8260a8e24cc5f081",
-    strip_prefix = "sqlite-amalgamation-3300100",
+    sha256 = "9c99955b21d2374f3a385d67a1f64cbacb1d4130947473d25c77ad609c03b4cd",
+    strip_prefix = "sqlite-amalgamation-3390400",
     system_build_file = clean_dep("//ml_metadata/third_party/systemlibs:sqlite.BUILD"),
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2019/sqlite-amalgamation-3300100.zip",
-        "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip",
+        "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2022/sqlite-amalgamation-3390400.zip",
+        "https://www.sqlite.org/2022/sqlite-amalgamation-3390400.zip",
     ],
 )
 
@@ -71,63 +96,41 @@ http_archive(
 
 http_archive(
     name = "bazel_skylib",
-    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
     ],
 )
 
 # Needed by abseil-py by zetasql.
 http_archive(
     name = "six_archive",
     urls = [
-        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.16.0.tar.gz",
+        "https://pypi.python.org/packages/source/s/six/six-1.16.0.tar.gz",
     ],
-    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
-    strip_prefix = "six-1.10.0",
+    sha256 = "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+    strip_prefix = "six-1.16.0",
     build_file = "//ml_metadata/third_party:six.BUILD"
 )
 
-PROTOBUF_COMMIT = "fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a" # 3.13.0
-http_archive(
-    name = "com_google_protobuf",
-    sha256 = "e589e39ef46fb2b3b476b3ca355bd324e5984cbdfac19f0e1625f0042e99c276",
-    strip_prefix = "protobuf-%s" % PROTOBUF_COMMIT,
-    urls = [
-        "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/protobuf/archive/%s.tar.gz" % PROTOBUF_COMMIT,
-        "https://github.com/google/protobuf/archive/%s.tar.gz" % PROTOBUF_COMMIT,
-    ],
-)
-
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-
-protobuf_deps()
-
-# Needed by Protobuf.
-http_archive(
-    name = "zlib",
-    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932",
-    strip_prefix = "zlib-1.2.12",
-    urls = ["https://github.com/madler/zlib/archive/v1.2.12.tar.gz"],
-)
-
 # pybind11
 http_archive(
     name = "pybind11",
+    sha256 = "eacf582fa8f696227988d08cfc46121770823839fe9e301a20fbce67e7cd70ec",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
-    strip_prefix = "pybind11-2.4.3",
-    urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"],
+    strip_prefix = "pybind11-2.10.0",
+    urls = ["https://github.com/pybind/pybind11/archive/v2.10.0.tar.gz"],
 )
 
 # Bazel rules for pybind11
+PYBIND11_BAZEL_COMMIT = "9a24c33cbdc510fa60ab7f5ffb7d80ab89272799" # up to date for pybind11 2.10.0
 http_archive(
     name = "pybind11_bazel",
-    strip_prefix = "pybind11_bazel-d5587e65fb8cbfc0015391a7616dc9c66f64a494",
-    url = "https://github.com/pybind/pybind11_bazel/archive/d5587e65fb8cbfc0015391a7616dc9c66f64a494.zip",
-    sha256 = "bf8e1f3ebde5ee37ad30c451377b03fbbe42b9d8f24c244aa8af2ccbaeca7e6c",
+    sha256 = "e1fe52ad3468629772c50c67a93449f235aed650a0fe8e89a22fbff285f677a1",
+    strip_prefix = "pybind11_bazel-%s" % PYBIND11_BAZEL_COMMIT,
+    url = "https://github.com/pybind/pybind11_bazel/archive/%s.zip" % PYBIND11_BAZEL_COMMIT,
 )
 load("@pybind11_bazel//:python_configure.bzl", "python_configure")
 python_configure(name = "local_config_python")
@@ -141,13 +144,12 @@ http_archive(
     strip_prefix = "re2-d1394506654e0a19a92f3d8921e26f7c3f4de969",
 )
 
-# gRPC. Official release 1.33.2. Name is required by Google APIs.
+# gRPC. Official release 1.48.0. Name is required by Google APIs.
 http_archive(
     name = "com_github_grpc_grpc",
-    sha256 = "2060769f2d4b0d3535ba594b2ab614d7f68a492f786ab94b4318788d45e3278a",
-    strip_prefix = "grpc-1.33.2",
-    patches = ["//ml_metadata/third_party:grpc.patch"],
-    urls = ["https://github.com/grpc/grpc/archive/v1.33.2.tar.gz"],
+    sha256 = "76900ab068da86378395a8e125b5cc43dfae671e09ff6462ddfef18676e2165a",
+    strip_prefix = "grpc-1.50.0",
+    urls = ["https://github.com/grpc/grpc/archive/v1.50.0.tar.gz"],
 )
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 grpc_deps()
@@ -164,34 +166,8 @@ bind(
     actual = "@com_github_grpc_grpc//:grpc++",
 )
 
-# Needed by gRPC.
-http_archive(
-    name = "build_bazel_rules_swift",
-    sha256 = "d0833bc6dad817a367936a5f902a0c11318160b5e80a20ece35fb85a5675c886",
-    strip_prefix = "rules_swift-3eeeb53cebda55b349d64c9fc144e18c5f7c0eb8",
-    urls = ["https://github.com/bazelbuild/rules_swift/archive/3eeeb53cebda55b349d64c9fc144e18c5f7c0eb8.tar.gz"],
-)
-
-http_archive(
-    name = "io_bazel_rules_go",
-    urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
-    ],
-    sha256 = "e88471aea3a3a4f19ec1310a55ba94772d087e9ce46e41ae38ecebe17935de7b",
-)
-
+# pulled in transitively by gRPC.
 load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
-
-http_archive(
-    name = "bazel_gazelle",
-    urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
-    ],
-    sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f",
-)
-
 load("@bazel_gazelle//:deps.bzl", "go_repository", "gazelle_dependencies")
 
 go_repository(
@@ -208,7 +184,7 @@ go_repository(
 
 go_rules_dependencies()
 
-go_register_toolchains()
+go_register_toolchains("host")
 
 gazelle_dependencies()
 
@@ -221,13 +197,12 @@ git_repository(
 )
 
 # BEGIN IFNDEF_WIN
-ZETASQL_COMMIT = "5ccb05880e72ab9ff75dd6b05d7b0acce53f1ea2" # 04/22/2021  # windows
+ZETASQL_COMMIT = "ac37cf5c0d80b5605176fc0f29e87b12f00be693" # 04/22/2021  # windows
 http_archive(  # windows
     name = "com_google_zetasql",  # windows
     urls = ["https://github.com/google/zetasql/archive/%s.zip" % ZETASQL_COMMIT],  # windows
     strip_prefix = "zetasql-%s" % ZETASQL_COMMIT,  # windows
-    # patches = ["//ml_metadata/third_party:zetasql.patch"],  # windows
-    sha256 = '4ca4e45f457926484822701ec15ca4d0172b01d7ce43c0b34c6f3ab98c95b241'  # windows
+    sha256 = "651a768cd51627f58aa6de7039aba9ddab22f4b0450521169800555269447840",
 )  # windows
 
 load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")  # windows
@@ -239,6 +214,10 @@ zetasql_deps_step_2(  # windows
     tools_deps = False,  # windows
     java_deps = False,  # windows
     testing_deps = False)  # windows
+load("@com_google_zetasql//bazel:zetasql_deps_step_3.bzl", "zetasql_deps_step_3")
+zetasql_deps_step_3()
+load("@com_google_zetasql//bazel:zetasql_deps_step_4.bzl", "zetasql_deps_step_4")
+zetasql_deps_step_4()
 # END IFNDEF_WIN
 
 

diff --git a/ml_metadata/.bazelversion b/ml_metadata/.bazelversion
@@ -11,4 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-3.7.2
+5.3.2
diff --git a/ml_metadata/libmysqlclient.BUILD b/ml_metadata/libmysqlclient.BUILD
@@ -34,8 +34,10 @@ genrule(
 )
 
 config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
+    name = "macos",
+    constraint_values = [
+        "@bazel_tools//platforms:osx",
+    ],
     visibility = ["//visibility:public"],
 )
 
@@ -45,13 +47,14 @@ cc_library(
         # plugins.
         "plugins/auth/my_auth.c",
         "plugins/auth/old_password.c",
+        "plugins/compress/c_zlib.c",
         "plugins/pvio/pvio_socket.c",
         # ssl.
         "libmariadb/secure/openssl.c",
         # core libmariadb
         "libmariadb/ma_array.c",
         "libmariadb/ma_charset.c",
-        "libmariadb/ma_hash.c",
+        "libmariadb/ma_hashtbl.c",
         "libmariadb/ma_net.c",
         "libmariadb/mariadb_charset.c",
         "libmariadb/ma_time.c",
@@ -91,6 +94,7 @@ cc_library(
         "-DLIBICONV_PLUG",
         "-DHAVE_OPENSSL",
         "-DHAVE_TLS",
+        "-D_XOPEN_SOURCE",
     ],
     includes = [
         "build/include/",
@@ -102,7 +106,7 @@ cc_library(
         "-ldl",
         "-lm",
     ] + select({
-        ":darwin": ["-liconv"],
+        ":macos": ["-liconv"],
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],

diff --git a/ml_metadata/metadata_store/BUILD b/ml_metadata/metadata_store/BUILD
@@ -71,7 +71,6 @@ cc_library(
         ":metadata_source",
         ":query_executor",
         "@com_google_protobuf//:protobuf",
-
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/status",

diff --git a/ml_metadata/metadata_store/pywrap/BUILD b/ml_metadata/metadata_store/pywrap/BUILD
@@ -36,6 +36,7 @@ ml_metadata_pybind_extension(
         "//ml_metadata/simple_types/proto:simple_types_proto",
         "//ml_metadata/metadata_store:simple_types_util",
         "@pybind11",
+        "@local_config_python//:python_headers",
     ],
 )
 

diff --git a/ml_metadata/ml_metadata.bzl b/ml_metadata/ml_metadata.bzl
@@ -91,7 +91,7 @@ def ml_metadata_proto_library_py(
         name = name,
         srcs = srcs,
         srcs_version = "PY2AND3",
-        deps = ["@com_google_protobuf//:protobuf_python"] + deps + oss_deps,
+        deps = ["@com_google_protobuf//:well_known_types_py_pb2"] + deps + oss_deps,
         default_runtime = "@com_google_protobuf//:protobuf_python",
         protoc = "@com_google_protobuf//:protoc",
         visibility = visibility,
@@ -262,6 +262,7 @@ def ml_metadata_pybind_extension(
             ],
             "//ml_metadata:windows": [],
             "//conditions:default": [
+                "-Wl,-Bsymbolic",
                 "-Wl,--version-script",
                 "$(location %s)" % version_script_file,
             ],
@@ -270,7 +271,7 @@ def ml_metadata_pybind_extension(
             exported_symbols_file,
             version_script_file,
         ],
-        features = ["-use_header_modules"],
+        features = ["-use_header_modules", "-parse_headers"],
         linkshared = 1,
         visibility = visibility,
     )

diff --git a/ml_metadata/move_generated_files.sh b/ml_metadata/move_generated_files.sh
@@ -54,8 +54,10 @@ function mlmd::move_generated_files() {
       ${BUILD_WORKSPACE_DIRECTORY}/${MLMD_EXTENSION}
   else
     MLMD_EXTENSION="ml_metadata/metadata_store/pywrap/metadata_store_extension.so"
-    cp -f ${BUILD_WORKSPACE_DIRECTORY}/bazel-bin/${MLMD_EXTENSION} \
-      ${BUILD_WORKSPACE_DIRECTORY}/${MLMD_EXTENSION}
+    DEST_FILE="${BUILD_WORKSPACE_DIRECTORY}/${MLMD_EXTENSION}"
+    # remove old .so file first to avoid catastrophic failure on macOS due to codesign.
+    rm -f "$DEST_FILE"
+    cp -f ${BUILD_WORKSPACE_DIRECTORY}/bazel-bin/${MLMD_EXTENSION} "${DEST_FILE}"
   fi
 
   chmod +w "${BUILD_WORKSPACE_DIRECTORY}/${MLMD_EXTENSION}"

diff --git a/ml_metadata/mysql_configure.bzl b/ml_metadata/mysql_configure.bzl
@@ -25,6 +25,7 @@ def mysql_configure():
         name = "libmysqlclient",
         build_file = "//ml_metadata:libmysqlclient.BUILD",
         workspace_file = "//ml_metadata:libmysqlclient.WORKSPACE",
+        patches = ["//ml_metadata/third_party:mariadbconnector.patch"],
         remote = "https://github.com/MariaDB/mariadb-connector-c.git",
-        tag = "v3.0.8-release",
+        tag = "v3.3.2",
     )