diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml new file mode 100644 index 000000000..cc529ce4c --- /dev/null +++ b/.github/workflows/cd-docs.yml @@ -0,0 +1,48 @@ +name: deploy-docs +on: + workflow_dispatch: + push: + branches: + - 'master' + pull_request: +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: | + setup.py + + - name: Save time for cache for mkdocs + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Caching + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install Dependencies + run: pip install -r requirements-docs.txt + + - name: Deploy to GitHub Pages + if: (github.event_name != 'pull_request') + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + mkdocs gh-deploy --force + + - name: Build docs to check for errors + run: mkdocs build + if: (github.event_name == 'pull_request') diff --git a/g3doc/get_started.md b/docs/index.md similarity index 95% rename from g3doc/get_started.md rename to docs/index.md index f9908a2f0..43e18a1ce 100644 --- a/g3doc/get_started.md +++ b/docs/index.md @@ -190,7 +190,7 @@ following list provides a non-exhaustive overview of some of the major benefits. within a range; find previous executions in a context with the same inputs. See the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) for +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) for an example that shows you how to use the MLMD API and the metadata store to retrieve lineage information. @@ -484,10 +484,11 @@ the steps together within a single transaction, and the transaction is rolled-back when an error occurs. The migration script is provided together with any schema-change commit and verified through testing. -Note: The migration DDLs in MySQL are not transactional. When using MySQL, there -should only be a single connection with the upgrade migration enabled to use the -old database. Take a backup of the database before upgrading to prevent -potential data losses. +!!! Note + The migration DDLs in MySQL are not transactional. When using MySQL, there + should only be a single connection with the upgrade migration enabled to use the + old database. Take a backup of the database before upgrading to prevent + potential data losses. ### Downgrade the database schema @@ -509,11 +510,12 @@ metadata_store.downgrade_schema(connection_config, downgrade_to_schema_version = 0) ``` -Note: When downgrading, MLMD prevents data loss as much as possible. However, -newer schema versions might be inherently more expressive and hence a downgrade -can introduce data loss. When using backends that do not support DDL -transactions (e.g., MySQL), the database should be backed up before downgrading -and the downgrade script should be the only MLMD connection to the database. +!!! Note + When downgrading, MLMD prevents data loss as much as possible. However, + newer schema versions might be inherently more expressive and hence a downgrade + can introduce data loss. When using backends that do not support DDL + transactions (e.g., MySQL), the database should be backed up before downgrading + and the downgrade script should be the only MLMD connection to the database. The list of `schema_version` used in MLMD releases are: @@ -555,7 +557,7 @@ ml-metadata (MLMD) | schema_version The MLMD library has a high-level API that you can readily use with your ML pipelines. See the -[MLMD API documentation](https://www.tensorflow.org/tfx/ml_metadata/api_docs/python/mlmd) +[MLMD API documentation](api/mlmd/) for more details. Check out @@ -564,5 +566,5 @@ to learn how to use MLMD declarative nodes filtering capabilities on properties and 1-hop neighborhood nodes. Also check out the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) to +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) to learn how to use MLMD to trace the lineage of your pipeline components. diff --git a/g3doc/images/mlmd_flow.png b/g3doc/images/mlmd_flow.png deleted file mode 100644 index 72b39b927..000000000 Binary files a/g3doc/images/mlmd_flow.png and /dev/null differ diff --git a/g3doc/images/mlmd_overview.png b/g3doc/images/mlmd_overview.png deleted file mode 100644 index a5e5e05ed..000000000 Binary files a/g3doc/images/mlmd_overview.png and /dev/null differ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..f9a4ba2c0 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,118 @@ +site_name: "ML Metadata" +repo_name: "ML Metadata" +repo_url: https://github.com/google/ml-metadata + +theme: + name: material + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + primary: custom + accent: custom + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + primary: custom + accent: custom + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + primary: custom + accent: custom + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to system preference + favicon: images/favicon.png + + features: + - content.code.copy + - content.code.select +plugins: + - search + - autorefs + - mkdocstrings: + default_handler: python + handlers: + python: + options: + show_source: true + show_root_heading: true + unwrap_annotated: true + show_symbol_type_toc: true + show_symbol_type_heading: true + merge_init_into_class: true + show_signature_annotations: true + separate_signature: true + signature_crossrefs: true + group_by_category: true + show_category_heading: true + inherited_members: true + show_submodules: true + show_object_full_path: false + show_root_full_path: true + docstring_section_style: "spacy" + summary: true + filters: + - "!^_" + - "^__init__$" + - "^__call__$" + - "!^logger" + - "!_test$" + extensions: + - griffe_inherited_docstrings + import: + - https://docs.python.org/3/objects.inv + - caption: + figure: + ignore_alt: true + +markdown_extensions: + - admonition + - attr_list + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + linenums: false + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.arithmatex: + generic: true + - md_in_html + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +extra_css: + - stylesheets/extra.css + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js + +watch: + - ml_metadata +nav: + - Guide: index.md + - Tutorial: https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/ + - API: + - mlmd: + - Overview: api/mlmd/index.md + - mlmd: api/mlmd/mlmd.md + - mlmd.errors: + - Overview: api/mlmd.errors/index.md + - mlmd.errors: api/mlmd.errors/mlmd.errors.md + - mlmd.proto: + - Overview: api/mlmd.proto/index.md + - mlmd.proto: api/mlmd.proto/mlmd.proto.md diff --git a/ml_metadata/__init__.py b/ml_metadata/__init__.py index 556b634e3..a38d69066 100644 --- a/ml_metadata/__init__.py +++ b/ml_metadata/__init__.py @@ -13,23 +13,30 @@ # limitations under the License. """Init module for ML Metadata.""" - # pylint: disable=g-import-not-at-top try: - from ml_metadata import proto + from ml_metadata import proto - # Import metadata_store API. - from ml_metadata.metadata_store import ( - ListOptions, - MetadataStore, - OrderByField, - downgrade_schema, - ) + # Import metadata_store API. + from ml_metadata.metadata_store import ( + ListOptions, + MetadataStore, + OrderByField, + downgrade_schema, + ) - # Import version string. - from ml_metadata.version import __version__ + # Import version string. + from ml_metadata.version import __version__ except ImportError as err: - import sys - sys.stderr.write(f'Error importing: {err}') -# pylint: enable=g-import-not-at-top + import sys # pylint: enable=g-import-not-at-top + + sys.stderr.write(f"Error importing: {err}") + + +__all__ = [ + "downgrade_schema", + "ListOptions", + "MetadataStore", + "OrderByField", +] diff --git a/ml_metadata/metadata_store/__init__.py b/ml_metadata/metadata_store/__init__.py index 931725f76..e4eaded87 100644 --- a/ml_metadata/metadata_store/__init__.py +++ b/ml_metadata/metadata_store/__init__.py @@ -18,3 +18,10 @@ OrderByField, downgrade_schema, ) + +__all__ = [ + "downgrade_schema", + "ListOptions", + "MetadataStore", + "OrderByField", +] diff --git a/ml_metadata/metadata_store/metadata_store.py b/ml_metadata/metadata_store/metadata_store.py index 4ea07d547..1bdc50b07 100644 --- a/ml_metadata/metadata_store/metadata_store.py +++ b/ml_metadata/metadata_store/metadata_store.py @@ -53,7 +53,13 @@ @enum.unique class OrderByField(enum.Enum): - """Defines the available fields to order results in ListOperations.""" + """Defines the available fields to order results in ListOperations. + + Attributes: + CREATE_TIME: `` + ID: `` + UPDATE_TIME: `` + """ CREATE_TIME = ( metadata_store_pb2.ListOperationOptions.OrderByField.Field.CREATE_TIME) @@ -1041,8 +1047,8 @@ def get_artifacts_and_types_by_artifact_ids( Returns: Artifacts with matching ids and ArtifactTypes which can be matched by - type_ids from Artifacts. Each ArtifactType contains id, name, - properties and custom_properties fields. + type_ids from Artifacts. Each ArtifactType contains id, name, + properties and custom_properties fields. """ del extra_options request = metadata_store_service_pb2.GetArtifactsByIDRequest( diff --git a/ml_metadata/metadata_store/mlmd_types.py b/ml_metadata/metadata_store/mlmd_types.py index 7fc918283..93535762a 100644 --- a/ml_metadata/metadata_store/mlmd_types.py +++ b/ml_metadata/metadata_store/mlmd_types.py @@ -69,8 +69,8 @@ def __init__(self, type_name: str): Raises: NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list; - It also raises the corresponding error from wrapped LoadSimpleTypes util - method. + It also raises the corresponding error from wrapped LoadSimpleTypes util + method. """ [types_str, error_message, status_code] = metadata_store_serialized.LoadSimpleTypes() diff --git a/ml_metadata/proto/__init__.py b/ml_metadata/proto/__init__.py index a1207012b..a2b7cb2e1 100644 --- a/ml_metadata/proto/__init__.py +++ b/ml_metadata/proto/__init__.py @@ -12,35 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. """ML Metadata proto module.""" + +# Connection configurations for different deployment. from ml_metadata.proto import ( - metadata_store_pb2, metadata_store_service_pb2, metadata_store_service_pb2_grpc, ) -# Connection configurations for different deployment. -ConnectionConfig = metadata_store_pb2.ConnectionConfig -MetadataStoreClientConfig = metadata_store_pb2.MetadataStoreClientConfig - # ML Metadata core data model concepts. -Artifact = metadata_store_pb2.Artifact -Execution = metadata_store_pb2.Execution -Context = metadata_store_pb2.Context - -Event = metadata_store_pb2.Event -Attribution = metadata_store_pb2.Attribution -Association = metadata_store_pb2.Association -ParentContext = metadata_store_pb2.ParentContext - -ArtifactType = metadata_store_pb2.ArtifactType -ExecutionType = metadata_store_pb2.ExecutionType -ContextType = metadata_store_pb2.ContextType - -FakeDatabaseConfig = metadata_store_pb2.FakeDatabaseConfig -MySQLDatabaseConfig = metadata_store_pb2.MySQLDatabaseConfig -SqliteMetadataSourceConfig = metadata_store_pb2.SqliteMetadataSourceConfig +from ml_metadata.proto.metadata_store_pb2 import ( + Artifact, + ArtifactType, + Association, + Attribution, + ConnectionConfig, + Context, + ContextType, + Event, + Execution, + ExecutionType, + FakeDatabaseConfig, + MetadataStoreClientConfig, + MySQLDatabaseConfig, + ParentContext, + SqliteMetadataSourceConfig, +) -del metadata_store_pb2 del metadata_store_service_pb2 del metadata_store_service_pb2_grpc @@ -115,3 +112,21 @@ MetadataStoreClientConfig.__doc__ = """ A connection configuration to use a MLMD server as the persistent backend. """ + +__all__ = [ + "ConnectionConfig", + "MetadataStoreClientConfig", + "Artifact", + "Execution", + "Context", + "Event", + "Attribution", + "Association", + "ParentContext", + "ArtifactType", + "ExecutionType", + "ContextType", + "FakeDatabaseConfig", + "MySQLDatabaseConfig", + "SqliteMetadataSourceConfig", +] diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 000000000..2c76c4f30 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,8 @@ +mkdocs +mkdocs-material +mkdocstrings[python] +griffe-inherited-docstrings +mkdocs-autorefs +black +mkdocs-jupyter +mkdocs-caption diff --git a/setup.py b/setup.py index 92e896c89..5b1ad7415 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,11 @@ def run(self): with open('README.md') as fp: _LONG_DESCRIPTION = fp.read() +# Get documentation build requirements +with open("requirements-docs.txt") as fp: + docs_reqs = fp.readlines() +docs_reqs = [req.replace("\n", "") for req in docs_reqs] + setup( name='ml-metadata', version=__version__, @@ -166,6 +171,8 @@ def run(self): ], extras_require={ 'lint': ['pre-commit'], + # TODO: Pin versions for docs + "docs": docs_reqs, }, python_requires='>=3.9,<4', packages=find_packages(),