From 79eb203f2f8a567ac6eb357f836262435e8e3081 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:45:58 -0700 Subject: [PATCH 01/31] Add mkdocs as optional dependency --- setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92e896c89..55a642216 100644 --- a/setup.py +++ b/setup.py @@ -166,7 +166,16 @@ def run(self): ], extras_require={ 'lint': ['pre-commit'], - }, + # TODO: Pin versions for docs + "docs": ["mkdocs", + "mkdocs-material", + "mkdocstrings[python]", + "griffe-inherited-docstrings", + "mkdocs-autorefs", + "black", + "mkdocs-caption" + ] + } python_requires='>=3.9,<4', packages=find_packages(), include_package_data=True, From 34d7b802bfcc7dc2ac49879fd8a37816bc66dcdb Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:49:00 -0700 Subject: [PATCH 02/31] Add `mkdocs.yml` along with all necessary files --- g3doc/api/metadata_store.md | 4 + g3doc/assets/tf_full_color_primary_icon.svg | 1 + g3doc/javascripts/mathjax.js | 19 ++++ g3doc/stylesheets/extra.css | 15 +++ mkdocs.yml | 112 ++++++++++++++++++++ 5 files changed, 151 insertions(+) create mode 100644 g3doc/api/metadata_store.md create mode 100644 g3doc/assets/tf_full_color_primary_icon.svg create mode 100644 g3doc/javascripts/mathjax.js create mode 100644 g3doc/stylesheets/extra.css create mode 100644 mkdocs.yml diff --git a/g3doc/api/metadata_store.md b/g3doc/api/metadata_store.md new file mode 100644 index 000000000..a212b392a --- /dev/null +++ b/g3doc/api/metadata_store.md @@ -0,0 +1,4 @@ +# Metadata Store + + +::: ml_metadata.metadata_store diff --git a/g3doc/assets/tf_full_color_primary_icon.svg b/g3doc/assets/tf_full_color_primary_icon.svg new file mode 100644 index 000000000..3e7247778 --- /dev/null +++ b/g3doc/assets/tf_full_color_primary_icon.svg @@ -0,0 +1 @@ +FullColorPrimary Icon \ No newline at end of file diff --git a/g3doc/javascripts/mathjax.js b/g3doc/javascripts/mathjax.js new file mode 100644 index 000000000..0be88e041 --- /dev/null +++ b/g3doc/javascripts/mathjax.js @@ -0,0 +1,19 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex" + } +}; + +document$.subscribe(() => { + MathJax.startup.output.clearCache() + MathJax.typesetClear() + MathJax.texReset() + MathJax.typesetPromise() +}) diff --git a/g3doc/stylesheets/extra.css b/g3doc/stylesheets/extra.css new file mode 100644 index 000000000..e734efefd --- /dev/null +++ b/g3doc/stylesheets/extra.css @@ -0,0 +1,15 @@ +:root { + --md-primary-fg-color: #FFA800; + --md-primary-fg-color--light: #CCCCCC; + --md-primary-fg-color--dark: #425066; +} + +.video-wrapper { + max-width: 240px; + display: flex; + flex-direction: row; +} +.video-wrapper > iframe { + width: 100%; + aspect-ratio: 16 / 9; +} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..bb540a4df --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,112 @@ +site_name: "ML Metadata" +repo_name: "Tensorflow ML Metadata" +repo_url: https://github.com/google/ml-metadata + +docs_dir: g3doc + +theme: + name: material + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + primary: custom + accent: custom + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + primary: custom + accent: custom + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + primary: custom + accent: custom + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to system preference + logo: assets/tf_full_color_primary_icon.svg + favicon: assets/tf_full_color_primary_icon.svg + + features: + - content.code.copy + - content.code.select +plugins: + - search + - autorefs + - mkdocstrings: + default_handler: python + handlers: + python: + options: + show_source: true + show_root_heading: true + unwrap_annotated: true + show_symbol_type_toc: true + show_symbol_type_heading: true + merge_init_into_class: true + show_signature_annotations: true + separate_signature: true + signature_crossrefs: true + group_by_category: true + show_category_heading: true + inherited_members: true + show_submodules: true + show_object_full_path: false + show_root_full_path: true + docstring_section_style: "spacy" + summary: true + filters: + - "!^_" + - "^__init__$" + - "^__call__$" + - "!^logger" + extensions: + - griffe_inherited_docstrings + import: + - https://docs.python.org/3/objects.inv + - caption: + figure: + ignore_alt: true + +markdown_extensions: + - admonition + - attr_list + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + linenums: false + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.arithmatex: + generic: true + - md_in_html + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +extra_css: + - stylesheets/extra.css + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js + +watch: + - ml_metadata +nav: + - Getting Started: get_started.md + + - API: + - Metadata Store: api/metadata_store.md From 0d9e6d6324740d6543bda15ddd09512b73ac186b Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:52:12 -0700 Subject: [PATCH 03/31] Move icon to `images` folder --- g3doc/{assets => images}/tf_full_color_primary_icon.svg | 0 mkdocs.yml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename g3doc/{assets => images}/tf_full_color_primary_icon.svg (100%) diff --git a/g3doc/assets/tf_full_color_primary_icon.svg b/g3doc/images/tf_full_color_primary_icon.svg similarity index 100% rename from g3doc/assets/tf_full_color_primary_icon.svg rename to g3doc/images/tf_full_color_primary_icon.svg diff --git a/mkdocs.yml b/mkdocs.yml index bb540a4df..bc3ee5db9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,8 +32,8 @@ theme: toggle: icon: material/brightness-4 name: Switch to system preference - logo: assets/tf_full_color_primary_icon.svg - favicon: assets/tf_full_color_primary_icon.svg + logo: images/tf_full_color_primary_icon.svg + favicon: images/tf_full_color_primary_icon.svg features: - content.code.copy From 15a7747bf4735db7f184ff2e19835413f1ca37a7 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:53:06 -0700 Subject: [PATCH 04/31] Add workflow to deploy docs --- .github/workflows/cd-docs.yml | 45 ++++++++++++++++++++++++++ ml_metadata/metadata_store/__init__.py | 7 ++++ 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/cd-docs.yml diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml new file mode 100644 index 000000000..9a211fb34 --- /dev/null +++ b/.github/workflows/cd-docs.yml @@ -0,0 +1,45 @@ +name: deploy-docs +on: + workflow_dispatch: + push: + # Uncomment these lines before merge + # branches: + # - master +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + cache-dependency-path: | + setup.py + + - name: Save time for cache for mkdocs + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Caching + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install Dependencies + run: pip install mkdocs mkdocs-material mkdocstrings[python] griffe-inherited-docstrings mkdocs-autorefs black mkdocs-jupyter mkdocs-caption + + - name: Deploy to GitHub Pages + run: mkdocs gh-deploy --force diff --git a/ml_metadata/metadata_store/__init__.py b/ml_metadata/metadata_store/__init__.py index 931725f76..e4eaded87 100644 --- a/ml_metadata/metadata_store/__init__.py +++ b/ml_metadata/metadata_store/__init__.py @@ -18,3 +18,10 @@ OrderByField, downgrade_schema, ) + +__all__ = [ + "downgrade_schema", + "ListOptions", + "MetadataStore", + "OrderByField", +] From d6a4e361def6fd13360dc3ba25bdb7ab2e09ad6f Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:09:43 -0700 Subject: [PATCH 05/31] Ignore test modules --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index bc3ee5db9..91afe78c5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,6 +68,7 @@ plugins: - "^__init__$" - "^__call__$" - "!^logger" + - "!_test$" extensions: - griffe_inherited_docstrings import: From 874787d1ee39c87b1035f282eeb031b4d091223c Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:41:12 -0700 Subject: [PATCH 06/31] Fix newline issues in "Raises" and "Returns" sections --- ml_metadata/metadata_store/metadata_store.py | 4 ++-- ml_metadata/metadata_store/mlmd_types.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ml_metadata/metadata_store/metadata_store.py b/ml_metadata/metadata_store/metadata_store.py index 4ea07d547..96cfff948 100644 --- a/ml_metadata/metadata_store/metadata_store.py +++ b/ml_metadata/metadata_store/metadata_store.py @@ -1040,8 +1040,8 @@ def get_artifacts_and_types_by_artifact_ids( extra_options: ExtraOptions instance. Returns: - Artifacts with matching ids and ArtifactTypes which can be matched by - type_ids from Artifacts. Each ArtifactType contains id, name, + Artifacts with matching ids and ArtifactTypes which can be matched by\ + type_ids from Artifacts. Each ArtifactType contains id, name,\ properties and custom_properties fields. """ del extra_options diff --git a/ml_metadata/metadata_store/mlmd_types.py b/ml_metadata/metadata_store/mlmd_types.py index 7fc918283..a9da4320f 100644 --- a/ml_metadata/metadata_store/mlmd_types.py +++ b/ml_metadata/metadata_store/mlmd_types.py @@ -68,8 +68,8 @@ def __init__(self, type_name: str): type_name: name of the desired system type. Raises: - NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list; - It also raises the corresponding error from wrapped LoadSimpleTypes util + NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list;\ + It also raises the corresponding error from wrapped LoadSimpleTypes util\ method. """ [types_str, error_message, From f24501d8a951da235d993e2503c482f9b20eb156 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:11:33 -0700 Subject: [PATCH 07/31] Remove `.md` --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 91afe78c5..ed52f5ba3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,7 +107,7 @@ extra_javascript: watch: - ml_metadata nav: - - Getting Started: get_started.md + - Getting Started: get_started - API: - - Metadata Store: api/metadata_store.md + - Metadata Store: api/metadata_store From 476ee22f1908e6cb24644c41ac063ada9e8c871c Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:19:13 -0700 Subject: [PATCH 08/31] Add empty `index.md` --- g3doc/api/metadata_store.md | 4 -- g3doc/api/mlmd.errors/mlmd.errors.md | 3 ++ g3doc/api/mlmd.errors/root.md | 0 g3doc/api/mlmd.proto/mlmd.proto.md | 5 +++ g3doc/api/mlmd.proto/root.md | 0 g3doc/api/mlmd/mlmd.md | 5 +++ g3doc/api/mlmd/root.md | 1 + g3doc/index.md | 0 mkdocs.yml | 12 ++++- ml_metadata/__init__.py | 35 +++++++++------ ml_metadata/proto/__init__.py | 66 ++++++++++++++++------------ 11 files changed, 84 insertions(+), 47 deletions(-) delete mode 100644 g3doc/api/metadata_store.md create mode 100644 g3doc/api/mlmd.errors/mlmd.errors.md create mode 100644 g3doc/api/mlmd.errors/root.md create mode 100644 g3doc/api/mlmd.proto/mlmd.proto.md create mode 100644 g3doc/api/mlmd.proto/root.md create mode 100644 g3doc/api/mlmd/mlmd.md create mode 100644 g3doc/api/mlmd/root.md create mode 100644 g3doc/index.md diff --git a/g3doc/api/metadata_store.md b/g3doc/api/metadata_store.md deleted file mode 100644 index a212b392a..000000000 --- a/g3doc/api/metadata_store.md +++ /dev/null @@ -1,4 +0,0 @@ -# Metadata Store - - -::: ml_metadata.metadata_store diff --git a/g3doc/api/mlmd.errors/mlmd.errors.md b/g3doc/api/mlmd.errors/mlmd.errors.md new file mode 100644 index 000000000..87c3dc594 --- /dev/null +++ b/g3doc/api/mlmd.errors/mlmd.errors.md @@ -0,0 +1,3 @@ +# mlmd.errors + +::: ml_metadata.errors diff --git a/g3doc/api/mlmd.errors/root.md b/g3doc/api/mlmd.errors/root.md new file mode 100644 index 000000000..e69de29bb diff --git a/g3doc/api/mlmd.proto/mlmd.proto.md b/g3doc/api/mlmd.proto/mlmd.proto.md new file mode 100644 index 000000000..1af4e40d8 --- /dev/null +++ b/g3doc/api/mlmd.proto/mlmd.proto.md @@ -0,0 +1,5 @@ +# mlmd.proto + +::: ml_metadata.proto + options: + show_if_no_docstring: true diff --git a/g3doc/api/mlmd.proto/root.md b/g3doc/api/mlmd.proto/root.md new file mode 100644 index 000000000..e69de29bb diff --git a/g3doc/api/mlmd/mlmd.md b/g3doc/api/mlmd/mlmd.md new file mode 100644 index 000000000..7ac8b930f --- /dev/null +++ b/g3doc/api/mlmd/mlmd.md @@ -0,0 +1,5 @@ +# mlmd + +::: ml_metadata + options: + show_submodules: false diff --git a/g3doc/api/mlmd/root.md b/g3doc/api/mlmd/root.md new file mode 100644 index 000000000..4ce31d443 --- /dev/null +++ b/g3doc/api/mlmd/root.md @@ -0,0 +1 @@ +# MLMD diff --git a/g3doc/index.md b/g3doc/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/mkdocs.yml b/mkdocs.yml index ed52f5ba3..415a54cf7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,7 +107,15 @@ extra_javascript: watch: - ml_metadata nav: - - Getting Started: get_started + - Getting Started: get_started.md - API: - - Metadata Store: api/metadata_store + - mlmd: + - Overview: api/mlmd/root.md + - mlmd: api/mlmd/mlmd.md + - mlmd.errors: + - Overview: api/mlmd.errors/root.md + - mlmd.errors: api/mlmd.errors/mlmd.errors.md + - mlmd.proto: + - Overview: api/mlmd.proto/root.md + - mlmd.proto: api/mlmd.proto/mlmd.proto.md diff --git a/ml_metadata/__init__.py b/ml_metadata/__init__.py index 556b634e3..9b32d7d7c 100644 --- a/ml_metadata/__init__.py +++ b/ml_metadata/__init__.py @@ -13,23 +13,30 @@ # limitations under the License. """Init module for ML Metadata.""" - # pylint: disable=g-import-not-at-top try: - from ml_metadata import proto + from ml_metadata import proto - # Import metadata_store API. - from ml_metadata.metadata_store import ( - ListOptions, - MetadataStore, - OrderByField, - downgrade_schema, - ) + # Import metadata_store API. + from ml_metadata.metadata_store import ( + ListOptions, + MetadataStore, + OrderByField, + downgrade_schema, + ) - # Import version string. - from ml_metadata.version import __version__ + # Import version string. + from ml_metadata.version import __version__ except ImportError as err: - import sys - sys.stderr.write(f'Error importing: {err}') -# pylint: enable=g-import-not-at-top + import sys # pylint: enable=g-import-not-at-top + + sys.stderr.write(f"Error importing: {err}") + + +__all__ = [ + "downgrade_schema", + "ListOptions", + "Metadata", + "OrderByField", +] diff --git a/ml_metadata/proto/__init__.py b/ml_metadata/proto/__init__.py index a1207012b..d033691c6 100644 --- a/ml_metadata/proto/__init__.py +++ b/ml_metadata/proto/__init__.py @@ -12,37 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. """ML Metadata proto module.""" -from ml_metadata.proto import ( - metadata_store_pb2, - metadata_store_service_pb2, - metadata_store_service_pb2_grpc, -) # Connection configurations for different deployment. -ConnectionConfig = metadata_store_pb2.ConnectionConfig -MetadataStoreClientConfig = metadata_store_pb2.MetadataStoreClientConfig +from ml_metadata.proto.metadata_store_pb2 import ( + ConnectionConfig, + MetadataStoreClientConfig, +) # ML Metadata core data model concepts. -Artifact = metadata_store_pb2.Artifact -Execution = metadata_store_pb2.Execution -Context = metadata_store_pb2.Context - -Event = metadata_store_pb2.Event -Attribution = metadata_store_pb2.Attribution -Association = metadata_store_pb2.Association -ParentContext = metadata_store_pb2.ParentContext - -ArtifactType = metadata_store_pb2.ArtifactType -ExecutionType = metadata_store_pb2.ExecutionType -ContextType = metadata_store_pb2.ContextType - -FakeDatabaseConfig = metadata_store_pb2.FakeDatabaseConfig -MySQLDatabaseConfig = metadata_store_pb2.MySQLDatabaseConfig -SqliteMetadataSourceConfig = metadata_store_pb2.SqliteMetadataSourceConfig - -del metadata_store_pb2 -del metadata_store_service_pb2 -del metadata_store_service_pb2_grpc +from ml_metadata.proto.metadata_store_pb2 import ( + Artifact, + Execution, + Context, + Event, + Attribution, + Association, + ParentContext, + ArtifactType, + ExecutionType, + ContextType, + FakeDatabaseConfig, + MySQLDatabaseConfig, + SqliteMetadataSourceConfig, +) +import ml_metadata.proto.metadata_store_service_pb2 as _ +import ml_metadata.proto.metadata_store_service_pb2_grpc as _ Artifact.__doc__ = """ An artifact represents an input or an output of individual steps in a ML @@ -115,3 +109,21 @@ MetadataStoreClientConfig.__doc__ = """ A connection configuration to use a MLMD server as the persistent backend. """ + +__all__ = [ + "ConnectionConfig", + "MetadataStoreClientConfig", + "Artifact", + "Execution", + "Context", + "Event", + "Attribution", + "Association", + "ParentContext", + "ArtifactType", + "ExecutionType", + "ContextType", + "FakeDatabaseConfig", + "MySQLDatabaseConfig", + "SqliteMetadataSourceConfig", +] From e0c3c088436f6729dc7a7ec3a1169ee130f10962 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 18:40:48 -0700 Subject: [PATCH 09/31] Change favicon --- g3doc/images/favicon.png | Bin 0 -> 404 bytes mkdocs.yml | 2 +- ml_metadata/__init__.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 g3doc/images/favicon.png diff --git a/g3doc/images/favicon.png b/g3doc/images/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..00a8af6dabc6bd2705678240ab4a59398f490e80 GIT binary patch literal 404 zcmV;F0c-w=P)bWUp}xZee%_FF+E28Gr&LzQXejWB zFF+E26#x_I#XCg22?r-%Yj0PYEp7nRKcP5}NPfXW>d zAQS*X4L}sYk;G^K2?3k{u>_C=$Zl&HU|oxb9nw&M7hvPP797D5^YtB!^1A>lE_VEA zWtT8;3G)qN)hY)JtNaxtw(!6H4J$lDHv|41;8=iuIsgd)i~wGMGznl`3oC$`4v;qk yX22l>+=$W4)dl4JFr1FvuXH>&TgG!!vezfc^{4$!^Uogu0000 Date: Thu, 29 Aug 2024 18:53:33 -0700 Subject: [PATCH 10/31] Add attributes to docstring --- ml_metadata/metadata_store/metadata_store.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ml_metadata/metadata_store/metadata_store.py b/ml_metadata/metadata_store/metadata_store.py index 96cfff948..9e0f7f7ea 100644 --- a/ml_metadata/metadata_store/metadata_store.py +++ b/ml_metadata/metadata_store/metadata_store.py @@ -53,7 +53,13 @@ @enum.unique class OrderByField(enum.Enum): - """Defines the available fields to order results in ListOperations.""" + """Defines the available fields to order results in ListOperations. + + Attributes: + CREATE_TIME: `` + ID: `` + UPDATE_TIME: `` + """ CREATE_TIME = ( metadata_store_pb2.ListOperationOptions.OrderByField.Field.CREATE_TIME) From c3131accdf03e3db7ca3f002beea89d1166f5d59 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:41:40 -0700 Subject: [PATCH 11/31] Temporary fix for unused import --- ml_metadata/proto/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ml_metadata/proto/__init__.py b/ml_metadata/proto/__init__.py index d033691c6..b1eb7b4b9 100644 --- a/ml_metadata/proto/__init__.py +++ b/ml_metadata/proto/__init__.py @@ -35,8 +35,11 @@ MySQLDatabaseConfig, SqliteMetadataSourceConfig, ) -import ml_metadata.proto.metadata_store_service_pb2 as _ -import ml_metadata.proto.metadata_store_service_pb2_grpc as _ +from ml_metadata.proto import metadata_store_service_pb2 +from ml_metadata.proto import metadata_store_service_pb2_grpc + +del metadata_store_service_pb2 +del metadata_store_service_pb2_grpc Artifact.__doc__ = """ An artifact represents an input or an output of individual steps in a ML From 27905cae517829426dbf2ee1048776d23ba3b37e Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:15:48 -0700 Subject: [PATCH 12/31] Fill out Overview section --- g3doc/api/mlmd/root.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/g3doc/api/mlmd/root.md b/g3doc/api/mlmd/root.md index 4ce31d443..4a1890b72 100644 --- a/g3doc/api/mlmd/root.md +++ b/g3doc/api/mlmd/root.md @@ -1 +1,23 @@ -# MLMD +# mlmd + +Init module for ML Metadata. + +## Modules + +[`errors`][ml_metadata.errors] module: Exception types for MLMD errors. + +[`proto`][ml_metadata.proto] module: ML Metadata proto module. + +## Classes + +[`class ListOptions`][ml_metadata.ListOptions]: Defines the available options when listing nodes. + +[`class MetadataStore`][ml_metadata.MetadataStore]: A store for the metadata. + +[`class OrderByField`][ml_metadata.OrderByField]: Defines the available fields to order results in ListOperations. + +## Functions + +[`downgrade_schema(...)`][ml_metadata.downgrade_schema]: Downgrades the db specified in the connection config to a schema version. + + From 941dfe086d75b63e874e41223cfbf2b9628bac90 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 31 Aug 2024 19:19:58 -0700 Subject: [PATCH 13/31] Use indentation instead of slash character `\` --- ml_metadata/metadata_store/metadata_store.py | 6 +++--- ml_metadata/metadata_store/mlmd_types.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ml_metadata/metadata_store/metadata_store.py b/ml_metadata/metadata_store/metadata_store.py index 9e0f7f7ea..1bdc50b07 100644 --- a/ml_metadata/metadata_store/metadata_store.py +++ b/ml_metadata/metadata_store/metadata_store.py @@ -1046,9 +1046,9 @@ def get_artifacts_and_types_by_artifact_ids( extra_options: ExtraOptions instance. Returns: - Artifacts with matching ids and ArtifactTypes which can be matched by\ - type_ids from Artifacts. Each ArtifactType contains id, name,\ - properties and custom_properties fields. + Artifacts with matching ids and ArtifactTypes which can be matched by + type_ids from Artifacts. Each ArtifactType contains id, name, + properties and custom_properties fields. """ del extra_options request = metadata_store_service_pb2.GetArtifactsByIDRequest( diff --git a/ml_metadata/metadata_store/mlmd_types.py b/ml_metadata/metadata_store/mlmd_types.py index a9da4320f..93535762a 100644 --- a/ml_metadata/metadata_store/mlmd_types.py +++ b/ml_metadata/metadata_store/mlmd_types.py @@ -68,9 +68,9 @@ def __init__(self, type_name: str): type_name: name of the desired system type. Raises: - NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list;\ - It also raises the corresponding error from wrapped LoadSimpleTypes util\ - method. + NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list; + It also raises the corresponding error from wrapped LoadSimpleTypes util + method. """ [types_str, error_message, status_code] = metadata_store_serialized.LoadSimpleTypes() From a5100166dfb603f440686d23aaabd8cfd4404d50 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 31 Aug 2024 21:08:38 -0700 Subject: [PATCH 14/31] Add overview from `mlmd.errors` api docs --- g3doc/api/mlmd.errors/root.md | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/g3doc/api/mlmd.errors/root.md b/g3doc/api/mlmd.errors/root.md index e69de29bb..04562c7fa 100644 --- a/g3doc/api/mlmd.errors/root.md +++ b/g3doc/api/mlmd.errors/root.md @@ -0,0 +1,45 @@ +# mlmd.errors + +Exception types for MLMD errors. + +## Classes + +[`class AbortedError`][ml_metadata.errors.AbortedError]: The operation was aborted, typically due to a concurrent action. + +[`class AlreadyExistsError`][ml_metadata.errors.AlreadyExistsError]: Raised when an entity that we attempted to create already exists. + +[`class CancelledError`][ml_metadata.errors.CancelledError]: Raised when an operation or step is cancelled. + +[`class DataLossError`][ml_metadata.errors.DataLossError]: Raised when unrecoverable data loss or corruption is encountered. + +[`class DeadlineExceededError`][ml_metadata.errors.DeadlineExceededError]: Raised when a deadline expires before an operation could complete. + +[`class FailedPreconditionError`][ml_metadata.errors.FailedPreconditionError]: Raised when the system is not in a state to execute an operation. + +[`class InternalError`][ml_metadata.errors.InternalError]: Raised when the system experiences an internal error. + +[`class InvalidArgumentError`][ml_metadata.errors.InvalidArgumentError]: Raised when an operation receives an invalid argument. + +[`class NotFoundError`][ml_metadata.errors.NotFoundError]: Raised when a requested entity was not found. + +[`class OutOfRangeError`][ml_metadata.errors.OutOfRangeError]: Raised when an operation iterates past the valid input range. + +[`class PermissionDeniedError`][ml_metadata.errors.PermissionDeniedError]: Raised when the caller does not have permission to run an operation. + +[`class ResourceExhaustedError`][ml_metadata.errors.ResourceExhaustedError]: Some resource has been exhausted. + +[`class StatusError`][ml_metadata.errors.StatusError]: A general error class that cast maps Status to typed errors. + +[`class UnauthenticatedError`][ml_metadata.errors.UnauthenticatedError]: The request does not have valid authentication credentials. + +[`class UnavailableError`][ml_metadata.errors.UnavailableError]: Raised when the runtime is currently unavailable. + +[`class UnimplementedError`][ml_metadata.errors.UnimplementedError]: Raised when an operation has not been implemented. + +[`class UnknownError`][ml_metadata.errors.UnknownError]: Raised when an operation failed reason is unknown. + +## Functions + +[`exception_type_from_error_code(...)`][ml_metadata.errors.exception_type_from_error_code]: Returns error class w.r.t. the error_code. + +[`make_exception(...)`][ml_metadata.errors.make_exception]: Makes an exception with the MLMD error code. From 2649a31f351f476d95205235e8d4b253dc423b11 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 31 Aug 2024 21:10:29 -0700 Subject: [PATCH 15/31] Add `show_if_no_docstring` option to mlmd.errors --- g3doc/api/mlmd.errors/mlmd.errors.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/g3doc/api/mlmd.errors/mlmd.errors.md b/g3doc/api/mlmd.errors/mlmd.errors.md index 87c3dc594..3c1dada6c 100644 --- a/g3doc/api/mlmd.errors/mlmd.errors.md +++ b/g3doc/api/mlmd.errors/mlmd.errors.md @@ -1,3 +1,5 @@ # mlmd.errors ::: ml_metadata.errors + options: + show_if_no_docstring: true From f5b18b827927242eb30788d9f9e44161a1d9fb92 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 31 Aug 2024 21:12:51 -0700 Subject: [PATCH 16/31] Use spaces instead of tabs --- g3doc/api/mlmd.proto/mlmd.proto.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/g3doc/api/mlmd.proto/mlmd.proto.md b/g3doc/api/mlmd.proto/mlmd.proto.md index 1af4e40d8..e190a876c 100644 --- a/g3doc/api/mlmd.proto/mlmd.proto.md +++ b/g3doc/api/mlmd.proto/mlmd.proto.md @@ -1,5 +1,5 @@ # mlmd.proto ::: ml_metadata.proto - options: - show_if_no_docstring: true + options: + show_if_no_docstring: true From 4802bb27002d6433e0b22f975ede25099d95c018 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 31 Aug 2024 21:14:33 -0700 Subject: [PATCH 17/31] Add overview with links. Links do not work because API for `mlmd.proto` is not showing up yet. Links will work when this is fixed --- g3doc/api/mlmd.proto/root.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/g3doc/api/mlmd.proto/root.md b/g3doc/api/mlmd.proto/root.md index e69de29bb..a7cf83f20 100644 --- a/g3doc/api/mlmd.proto/root.md +++ b/g3doc/api/mlmd.proto/root.md @@ -0,0 +1,35 @@ +# mlmd.proto + +ML Metadata proto module. + +## Classes + +[`class Artifact`][ml_metadata.proto.Artifact]: An artifact represents an input or an output of individual steps in a ML workflow, e.g., a trained model, an input dataset, and evaluation metrics. + +[`class ArtifactType`][ml_metadata.proto.ArtifactType]: A user defined type about a collection of artifacts and their properties that are stored in the metadata store. + +[`class Association`][ml_metadata.proto.Association]: An association represents the relationship between executions and contexts. + +[`class Attribution`][ml_metadata.proto.Attribution]: An attribution represents the relationship between artifacts and contexts. + +[`class ConnectionConfig`][ml_metadata.proto.ConnectionConfig]: A connection configuration specifying the persistent backend to be used with MLMD. + +[`class Context`][ml_metadata.proto.Context]: A context defines a group of artifacts and/or executions. + +[`class ContextType`][ml_metadata.proto.ContextType]: A user defined type about a collection of contexts and their properties that are stored in the metadata store. + +[`class Event`][ml_metadata.proto.Event]: An event records the relationship between artifacts and executions. + +[`class Execution`][ml_metadata.proto.Execution]: An execution describes a component run or a step in an ML workflow along with its runtime parameters, e.g., a Trainer run, a data transformation step. + +[`class ExecutionType`][ml_metadata.proto.ExecutionType]: A user defined type about a collection of executions and their properties that are stored in the metadata store. + +[`class FakeDatabaseConfig`][ml_metadata.proto.FakeDatabaseConfig]: An in-memory database configuration for testing purpose. + +[`class MetadataStoreClientConfig`][ml_metadata.proto.MetadataStoreClientConfig]: A connection configuration to use a MLMD server as the persistent backend. + +[`class MySQLDatabaseConfig`][ml_metadata.proto.MySQLDatabaseConfig]: A connection configuration to use a MySQL db instance as a MLMD backend. + +[`class ParentContext`][ml_metadata.proto.ParentContext]: A parental context represents the relationship between contexts. + +[`class SqliteMetadataSourceConfig`][ml_metadata.proto.SqliteMetadataSourceConfig]: A connection configuration to use a Sqlite db file as a MLMD backend. From 47960a4d8d86e6bccc8e31f8ddf4f3e2bc398d7c Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:35:45 -0700 Subject: [PATCH 18/31] Add link to tutorial --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 6da433c02..727e52c7b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -108,7 +108,7 @@ watch: - ml_metadata nav: - Getting Started: get_started.md - + - Tutorial: https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/ - API: - mlmd: - Overview: api/mlmd/root.md From ead6757c2760d632340a4ec7d17ad9cb0338cec4 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:39:17 -0700 Subject: [PATCH 19/31] Fix repo name --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 727e52c7b..2cf697934 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: "ML Metadata" -repo_name: "Tensorflow ML Metadata" +repo_name: "ML Metadata" repo_url: https://github.com/google/ml-metadata docs_dir: g3doc From 18875d81c6f6e08be3508769b41576049c021ca7 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:56:26 -0700 Subject: [PATCH 20/31] Rename `root.md` to `index.md` --- g3doc/api/mlmd.errors/{root.md => index.md} | 0 g3doc/api/mlmd.proto/{root.md => index.md} | 0 g3doc/api/mlmd/{root.md => index.md} | 0 g3doc/get_started.md | 568 -------------------- g3doc/index.md | 568 ++++++++++++++++++++ mkdocs.yml | 8 +- 6 files changed, 572 insertions(+), 572 deletions(-) rename g3doc/api/mlmd.errors/{root.md => index.md} (100%) rename g3doc/api/mlmd.proto/{root.md => index.md} (100%) rename g3doc/api/mlmd/{root.md => index.md} (100%) delete mode 100644 g3doc/get_started.md diff --git a/g3doc/api/mlmd.errors/root.md b/g3doc/api/mlmd.errors/index.md similarity index 100% rename from g3doc/api/mlmd.errors/root.md rename to g3doc/api/mlmd.errors/index.md diff --git a/g3doc/api/mlmd.proto/root.md b/g3doc/api/mlmd.proto/index.md similarity index 100% rename from g3doc/api/mlmd.proto/root.md rename to g3doc/api/mlmd.proto/index.md diff --git a/g3doc/api/mlmd/root.md b/g3doc/api/mlmd/index.md similarity index 100% rename from g3doc/api/mlmd/root.md rename to g3doc/api/mlmd/index.md diff --git a/g3doc/get_started.md b/g3doc/get_started.md deleted file mode 100644 index f9908a2f0..000000000 --- a/g3doc/get_started.md +++ /dev/null @@ -1,568 +0,0 @@ -# ML Metadata - -[ML Metadata (MLMD)](https://github.com/google/ml-metadata) is a library for -recording and retrieving metadata associated with ML developer and data -scientist workflows. MLMD is an integral part of -[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx), but is designed so -that it can be used independently. - -Every run of a production ML pipeline generates metadata containing information -about the various pipeline components, their executions (e.g. training runs), -and resulting artifacts (e.g. trained models). In the event of unexpected -pipeline behavior or errors, this metadata can be leveraged to analyze the -lineage of pipeline components and debug issues. Think of this metadata as the -equivalent of logging in software development. - -MLMD helps you understand and analyze all the interconnected parts of your ML -pipeline instead of analyzing them in isolation and can help you answer -questions about your ML pipeline such as: - -* Which dataset did the model train on? -* What were the hyperparameters used to train the model? -* Which pipeline run created the model? -* Which training run led to this model? -* Which version of TensorFlow created this model? -* When was the failed model pushed? - -## Metadata store - -MLMD registers the following types of metadata in a database called the -**Metadata Store**. - -1. Metadata about the artifacts generated through the components/steps of your - ML pipelines -1. Metadata about the executions of these components/steps -1. Metadata about pipelines and associated lineage information - -The Metadata Store provides APIs to record and retrieve metadata to and from the -storage backend. The storage backend is pluggable and can be extended. MLMD -provides reference implementations for SQLite (which supports in-memory and -disk) and MySQL out of the box. - -This graphic shows a high-level overview of the various components that are part -of MLMD. - -![ML Metadata Overview](images/mlmd_overview.png) - -### Metadata storage backends and store connection configuration - -The `MetadataStore` object receives a connection configuration that corresponds -to the storage backend used. - -* **Fake Database** provides an in-memory DB (using SQLite) for fast - experimentation and local runs. The database is deleted when the store - object is destroyed. - -```python -from ml_metadata import metadata_store -from ml_metadata.proto import metadata_store_pb2 - -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.fake_database.SetInParent() # Sets an empty fake database proto. -store = metadata_store.MetadataStore(connection_config) -``` - -* **SQLite** reads and writes files from disk. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -connection_config.sqlite.connection_mode = 3 # READWRITE_OPENCREATE -store = metadata_store.MetadataStore(connection_config) -``` - -* **MySQL** connects to a MySQL server. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.mysql.host = '...' -connection_config.mysql.port = '...' -connection_config.mysql.database = '...' -connection_config.mysql.user = '...' -connection_config.mysql.password = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -Similarly, when using a MySQL instance with Google CloudSQL -([quickstart](https://cloud.google.com/sql/docs/mysql/quickstart), -[connect-overview](https://cloud.google.com/sql/docs/mysql/connect-overview)), -one could also use SSL option if applicable. - -```python -connection_config.mysql.ssl_options.key = '...' -connection_config.mysql.ssl_options.cert = '...' -connection_config.mysql.ssl_options.ca = '...' -connection_config.mysql.ssl_options.capath = '...' -connection_config.mysql.ssl_options.cipher = '...' -connection_config.mysql.ssl_options.verify_server_cert = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -* **PostgreSQL** connects to a PostgreSQL server. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.postgresql.host = '...' -connection_config.postgresql.port = '...' -connection_config.postgresql.user = '...' -connection_config.postgresql.password = '...' -connection_config.postgresql.dbname = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -Similarly, when using a PostgreSQL instance with Google CloudSQL -([quickstart](https://cloud.google.com/sql/docs/postgres/quickstart), -[connect-overview](https://cloud.google.com/sql/docs/postgres/connect-overview)), -one could also use SSL option if applicable. - -```python -connection_config.postgresql.ssloption.sslmode = '...' # disable, allow, verify-ca, verify-full, etc. -connection_config.postgresql.ssloption.sslcert = '...' -connection_config.postgresql.ssloption.sslkey = '...' -connection_config.postgresql.ssloption.sslpassword = '...' -connection_config.postgresql.ssloption.sslrootcert = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -## Data model - -The Metadata Store uses the following data model to record and retrieve metadata -from the storage backend. - -* `ArtifactType` describes an artifact's type and its properties that are - stored in the metadata store. You can register these types on-the-fly with - the metadata store in code, or you can load them in the store from a - serialized format. Once you register a type, its definition is available - throughout the lifetime of the store. -* An `Artifact` describes a specific instance of an `ArtifactType`, and its - properties that are written to the metadata store. -* An `ExecutionType` describes a type of component or step in a workflow, and - its runtime parameters. -* An `Execution` is a record of a component run or a step in an ML workflow - and the runtime parameters. An execution can be thought of as an instance of - an `ExecutionType`. Executions are recorded when you run an ML pipeline or - step. -* An `Event` is a record of the relationship between artifacts and executions. - When an execution happens, events record every artifact that was used by the - execution, and every artifact that was produced. These records allow for - lineage tracking throughout a workflow. By looking at all events, MLMD knows - what executions happened and what artifacts were created as a result. MLMD - can then recurse back from any artifact to all of its upstream inputs. -* A `ContextType` describes a type of conceptual group of artifacts and - executions in a workflow, and its structural properties. For example: - projects, pipeline runs, experiments, owners etc. -* A `Context` is an instance of a `ContextType`. It captures the shared - information within the group. For example: project name, changelist commit - id, experiment annotations etc. It has a user-defined unique name within its - `ContextType`. -* An `Attribution` is a record of the relationship between artifacts and - contexts. -* An `Association` is a record of the relationship between executions and - contexts. - -## MLMD Functionality - -Tracking the inputs and outputs of all components/steps in an ML workflow and -their lineage allows ML platforms to enable several important features. The -following list provides a non-exhaustive overview of some of the major benefits. - -* **List all Artifacts of a specific type.** Example: all Models that have - been trained. -* **Load two Artifacts of the same type for comparison.** Example: compare - results from two experiments. -* **Show a DAG of all related executions and their input and output artifacts - of a context.** Example: visualize the workflow of an experiment for - debugging and discovery. -* **Recurse back through all events to see how an artifact was created.** - Examples: see what data went into a model; enforce data retention plans. -* **Identify all artifacts that were created using a given artifact.** - Examples: see all Models trained from a specific dataset; mark models based - upon bad data. -* **Determine if an execution has been run on the same inputs before.** - Example: determine whether a component/step has already completed the same - work and the previous output can just be reused. -* **Record and query context of workflow runs.** Examples: track the owner and - changelist used for a workflow run; group the lineage by experiments; manage - artifacts by projects. -* **Declarative nodes filtering capabilities on properties and 1-hop - neighborhood nodes.** Examples: look for artifacts of a type and under some - pipeline context; return typed artifacts where a given property’s value is - within a range; find previous executions in a context with the same inputs. - -See the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) for -an example that shows you how to use the MLMD API and the metadata store to -retrieve lineage information. - -### Integrate ML Metadata into your ML Workflows - -If you are a platform developer interested in integrating MLMD into your system, -use the example workflow below to use the low-level MLMD APIs to track the -execution of a training task. You can also use higher-level Python APIs in -notebook environments to record experiment metadata. - -![ML Metadata Example Flow](images/mlmd_flow.png) - -1) Register artifact types - -```python -# Create ArtifactTypes, e.g., Data and Model -data_type = metadata_store_pb2.ArtifactType() -data_type.name = "DataSet" -data_type.properties["day"] = metadata_store_pb2.INT -data_type.properties["split"] = metadata_store_pb2.STRING -data_type_id = store.put_artifact_type(data_type) - -model_type = metadata_store_pb2.ArtifactType() -model_type.name = "SavedModel" -model_type.properties["version"] = metadata_store_pb2.INT -model_type.properties["name"] = metadata_store_pb2.STRING -model_type_id = store.put_artifact_type(model_type) - -# Query all registered Artifact types. -artifact_types = store.get_artifact_types() -``` - -2) Register execution types for all steps in the ML workflow - -```python -# Create an ExecutionType, e.g., Trainer -trainer_type = metadata_store_pb2.ExecutionType() -trainer_type.name = "Trainer" -trainer_type.properties["state"] = metadata_store_pb2.STRING -trainer_type_id = store.put_execution_type(trainer_type) - -# Query a registered Execution type with the returned id -[registered_type] = store.get_execution_types_by_id([trainer_type_id]) -``` - -3) Create an artifact of DataSet ArtifactType - -```python -# Create an input artifact of type DataSet -data_artifact = metadata_store_pb2.Artifact() -data_artifact.uri = 'path/to/data' -data_artifact.properties["day"].int_value = 1 -data_artifact.properties["split"].string_value = 'train' -data_artifact.type_id = data_type_id -[data_artifact_id] = store.put_artifacts([data_artifact]) - -# Query all registered Artifacts -artifacts = store.get_artifacts() - -# Plus, there are many ways to query the same Artifact -[stored_data_artifact] = store.get_artifacts_by_id([data_artifact_id]) -artifacts_with_uri = store.get_artifacts_by_uri(data_artifact.uri) -artifacts_with_conditions = store.get_artifacts( - list_options=mlmd.ListOptions( - filter_query='uri LIKE "%/data" AND properties.day.int_value > 0')) -``` - -4) Create an execution of the Trainer run - -```python -# Register the Execution of a Trainer run -trainer_run = metadata_store_pb2.Execution() -trainer_run.type_id = trainer_type_id -trainer_run.properties["state"].string_value = "RUNNING" -[run_id] = store.put_executions([trainer_run]) - -# Query all registered Execution -executions = store.get_executions_by_id([run_id]) -# Similarly, the same execution can be queried with conditions. -executions_with_conditions = store.get_executions( - list_options = mlmd.ListOptions( - filter_query='type = "Trainer" AND properties.state.string_value IS NOT NULL')) -``` - -5) Define the input event and read data - -```python -# Define the input event -input_event = metadata_store_pb2.Event() -input_event.artifact_id = data_artifact_id -input_event.execution_id = run_id -input_event.type = metadata_store_pb2.Event.DECLARED_INPUT - -# Record the input event in the metadata store -store.put_events([input_event]) -``` - -6) Declare the output artifact - -```python -# Declare the output artifact of type SavedModel -model_artifact = metadata_store_pb2.Artifact() -model_artifact.uri = 'path/to/model/file' -model_artifact.properties["version"].int_value = 1 -model_artifact.properties["name"].string_value = 'MNIST-v1' -model_artifact.type_id = model_type_id -[model_artifact_id] = store.put_artifacts([model_artifact]) -``` - -7) Record the output event - -```python -# Declare the output event -output_event = metadata_store_pb2.Event() -output_event.artifact_id = model_artifact_id -output_event.execution_id = run_id -output_event.type = metadata_store_pb2.Event.DECLARED_OUTPUT - -# Submit output event to the Metadata Store -store.put_events([output_event]) -``` - -8) Mark the execution as completed - -```python -trainer_run.id = run_id -trainer_run.properties["state"].string_value = "COMPLETED" -store.put_executions([trainer_run]) -``` - -9) Group artifacts and executions under a context using attributions and -assertions artifacts - -```python -# Create a ContextType, e.g., Experiment with a note property -experiment_type = metadata_store_pb2.ContextType() -experiment_type.name = "Experiment" -experiment_type.properties["note"] = metadata_store_pb2.STRING -experiment_type_id = store.put_context_type(experiment_type) - -# Group the model and the trainer run to an experiment. -my_experiment = metadata_store_pb2.Context() -my_experiment.type_id = experiment_type_id -# Give the experiment a name -my_experiment.name = "exp1" -my_experiment.properties["note"].string_value = "My first experiment." -[experiment_id] = store.put_contexts([my_experiment]) - -attribution = metadata_store_pb2.Attribution() -attribution.artifact_id = model_artifact_id -attribution.context_id = experiment_id - -association = metadata_store_pb2.Association() -association.execution_id = run_id -association.context_id = experiment_id - -store.put_attributions_and_associations([attribution], [association]) - -# Query the Artifacts and Executions that are linked to the Context. -experiment_artifacts = store.get_artifacts_by_context(experiment_id) -experiment_executions = store.get_executions_by_context(experiment_id) - -# You can also use neighborhood queries to fetch these artifacts and executions -# with conditions. -experiment_artifacts_with_conditions = store.get_artifacts( - list_options = mlmd.ListOptions( - filter_query=('contexts_a.type = "Experiment" AND contexts_a.name = "exp1"'))) -experiment_executions_with_conditions = store.get_executions( - list_options = mlmd.ListOptions( - filter_query=('contexts_a.id = {}'.format(experiment_id)))) -``` - -## Use MLMD with a remote gRPC server - -You can use MLMD with remote gRPC servers as shown below: - -* Start a server - -```bash -bazel run -c opt --define grpc_no_ares=true //ml_metadata/metadata_store:metadata_store_server -``` - -By default, the server uses a fake in-memory db per request and does not persist -the metadata across calls. It can also be configured with a MLMD -`MetadataStoreServerConfig` to use SQLite files or MySQL instances. The config -can be stored in a text protobuf file and passed to the binary with -`--metadata_store_server_config_file=path_to_the_config_file`. - -An example `MetadataStoreServerConfig` file in text protobuf format: - -```textpb -connection_config { - sqlite { - filename_uri: '/tmp/test_db' - connection_mode: READWRITE_OPENCREATE - } -} -``` - -* Create the client stub and use it in Python - -```python -from grpc import insecure_channel -from ml_metadata.proto import metadata_store_pb2 -from ml_metadata.proto import metadata_store_service_pb2 -from ml_metadata.proto import metadata_store_service_pb2_grpc - -channel = insecure_channel('localhost:8080') -stub = metadata_store_service_pb2_grpc.MetadataStoreServiceStub(channel) -``` - -* Use MLMD with RPC calls - -```python -# Create ArtifactTypes, e.g., Data and Model -data_type = metadata_store_pb2.ArtifactType() -data_type.name = "DataSet" -data_type.properties["day"] = metadata_store_pb2.INT -data_type.properties["split"] = metadata_store_pb2.STRING - -request = metadata_store_service_pb2.PutArtifactTypeRequest() -request.all_fields_match = True -request.artifact_type.CopyFrom(data_type) -stub.PutArtifactType(request) - -model_type = metadata_store_pb2.ArtifactType() -model_type.name = "SavedModel" -model_type.properties["version"] = metadata_store_pb2.INT -model_type.properties["name"] = metadata_store_pb2.STRING - -request.artifact_type.CopyFrom(model_type) -stub.PutArtifactType(request) -``` - -## Upgrade the MLMD library - -When using a new MLMD release or your own build with an existing MLMD database, -there may be changes to the database schema. Unless a breaking change is -explicitly mentioned in the release note, all MLMD database schema changes are -transparent for the MLMD API users. If there is a breaking change notice, then -old databases can still be upgraded to use the new MLMD library. - -When the MLMD library connects to the database, it compares the expected schema -version of the MLMD library (`library_version`) with the schema version -(`db_version`) recorded in the given database. By default, MLMD will check the -compatibility and raise errors when the versions are incompatible. - -* If `library_version` is compatible with `db_version`, nothing happens. -* If `library_version` is newer than `db_version`, and auto-migration is not - enabled, then MLMD raises a failed precondition error with the following - message: - - ``` - MLMD database version $db_version is older than library version - $library_version. Schema migration is disabled. Please upgrade the - database then use the library version; or switch to a older library - version to use the current database. - ``` - -* If `library_version` is older than `db_version`, by default MLMD library - returns errors to prevent any data loss. In this case, you should upgrade - the library version before using that database. - -### Upgrade the database schema - -MLMD provides utilities to upgrade the database version. - -For example, when connecting to a backend with a Python library: - -```python {highlight="range:enable,True"} -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -store = metadata_store.MetadataStore(connection_config, - enable_upgrade_migration=True) -``` - -Or when using gRPC server, set the MetadataStoreServerConfig as follows: - -```python -connection_config { - ... -} -migration_options { - enable_upgrade_migration: true -} -``` - -MLMD then evolves the database by executing a series of migration scripts. If -the backend supports DDL queries within a transaction (e.g., SQLite), MLMD runs -the steps together within a single transaction, and the transaction is -rolled-back when an error occurs. The migration script is provided together with -any schema-change commit and verified through testing. - -Note: The migration DDLs in MySQL are not transactional. When using MySQL, there -should only be a single connection with the upgrade migration enabled to use the -old database. Take a backup of the database before upgrading to prevent -potential data losses. - -### Downgrade the database schema - -A misconfiguration in the deployment of MLMD may cause an accidental upgrade, -e.g., when you tries out a new version of the library and accidentally connect -to the production instance of MLMD and upgrade the database. To recover from -these situations, MLMD provides a downgrade feature. During connection, if the -migration options specify the `downgrade_to_schema_version`, MLMD will run a -downgrade transaction to revert the schema version and migrate the data, then -terminate the connection. Once the downgrade is done, use the older version of -the library to connect to the database. - -For example: - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -metadata_store.downgrade_schema(connection_config, - downgrade_to_schema_version = 0) -``` - -Note: When downgrading, MLMD prevents data loss as much as possible. However, -newer schema versions might be inherently more expressive and hence a downgrade -can introduce data loss. When using backends that do not support DDL -transactions (e.g., MySQL), the database should be backed up before downgrading -and the downgrade script should be the only MLMD connection to the database. - -The list of `schema_version` used in MLMD releases are: - -ml-metadata (MLMD) | schema_version ------------------- | -------------- -1.16.0 | 10 -1.15.0 | 10 -1.14.0 | 10 -1.13.1 | 10 -1.13.0 | 10 -1.12.0 | 10 -1.11.0 | 10 -1.10.0 | 8 -1.9.0 | 8 -1.8.0 | 8 -1.7.0 | 8 -1.6.0 | 7 -1.5.0 | 7 -1.4.0 | 7 -1.3.0 | 7 -1.2.0 | 7 -1.1.0 | 7 -1.0.0 | 6 -0.30.0 | 6 -0.29.0 | 6 -0.28.0 | 6 -0.27.0 | 6 -0.26.0 | 6 -0.25.1 | 6 -0.24.0 | 5 -0.23.0 | 5 -0.22.1 | 5 -0.21.2 | 4 -0.15.2 | 4 -0.14.0 | 4 -0.13.2 | 0 - -## Resources - -The MLMD library has a high-level API that you can readily use with your ML -pipelines. See the -[MLMD API documentation](https://www.tensorflow.org/tfx/ml_metadata/api_docs/python/mlmd) -for more details. - -Check out -[MLMD Declarative Nodes Filtering](https://github.com/google/ml-metadata/blob/v1.2.0/ml_metadata/proto/metadata_store.proto#L708-L786) -to learn how to use MLMD declarative nodes filtering capabilities on properties -and 1-hop neighborhood nodes. - -Also check out the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) to -learn how to use MLMD to trace the lineage of your pipeline components. diff --git a/g3doc/index.md b/g3doc/index.md index e69de29bb..f9908a2f0 100644 --- a/g3doc/index.md +++ b/g3doc/index.md @@ -0,0 +1,568 @@ +# ML Metadata + +[ML Metadata (MLMD)](https://github.com/google/ml-metadata) is a library for +recording and retrieving metadata associated with ML developer and data +scientist workflows. MLMD is an integral part of +[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx), but is designed so +that it can be used independently. + +Every run of a production ML pipeline generates metadata containing information +about the various pipeline components, their executions (e.g. training runs), +and resulting artifacts (e.g. trained models). In the event of unexpected +pipeline behavior or errors, this metadata can be leveraged to analyze the +lineage of pipeline components and debug issues. Think of this metadata as the +equivalent of logging in software development. + +MLMD helps you understand and analyze all the interconnected parts of your ML +pipeline instead of analyzing them in isolation and can help you answer +questions about your ML pipeline such as: + +* Which dataset did the model train on? +* What were the hyperparameters used to train the model? +* Which pipeline run created the model? +* Which training run led to this model? +* Which version of TensorFlow created this model? +* When was the failed model pushed? + +## Metadata store + +MLMD registers the following types of metadata in a database called the +**Metadata Store**. + +1. Metadata about the artifacts generated through the components/steps of your + ML pipelines +1. Metadata about the executions of these components/steps +1. Metadata about pipelines and associated lineage information + +The Metadata Store provides APIs to record and retrieve metadata to and from the +storage backend. The storage backend is pluggable and can be extended. MLMD +provides reference implementations for SQLite (which supports in-memory and +disk) and MySQL out of the box. + +This graphic shows a high-level overview of the various components that are part +of MLMD. + +![ML Metadata Overview](images/mlmd_overview.png) + +### Metadata storage backends and store connection configuration + +The `MetadataStore` object receives a connection configuration that corresponds +to the storage backend used. + +* **Fake Database** provides an in-memory DB (using SQLite) for fast + experimentation and local runs. The database is deleted when the store + object is destroyed. + +```python +from ml_metadata import metadata_store +from ml_metadata.proto import metadata_store_pb2 + +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.fake_database.SetInParent() # Sets an empty fake database proto. +store = metadata_store.MetadataStore(connection_config) +``` + +* **SQLite** reads and writes files from disk. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +connection_config.sqlite.connection_mode = 3 # READWRITE_OPENCREATE +store = metadata_store.MetadataStore(connection_config) +``` + +* **MySQL** connects to a MySQL server. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.mysql.host = '...' +connection_config.mysql.port = '...' +connection_config.mysql.database = '...' +connection_config.mysql.user = '...' +connection_config.mysql.password = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +Similarly, when using a MySQL instance with Google CloudSQL +([quickstart](https://cloud.google.com/sql/docs/mysql/quickstart), +[connect-overview](https://cloud.google.com/sql/docs/mysql/connect-overview)), +one could also use SSL option if applicable. + +```python +connection_config.mysql.ssl_options.key = '...' +connection_config.mysql.ssl_options.cert = '...' +connection_config.mysql.ssl_options.ca = '...' +connection_config.mysql.ssl_options.capath = '...' +connection_config.mysql.ssl_options.cipher = '...' +connection_config.mysql.ssl_options.verify_server_cert = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +* **PostgreSQL** connects to a PostgreSQL server. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.postgresql.host = '...' +connection_config.postgresql.port = '...' +connection_config.postgresql.user = '...' +connection_config.postgresql.password = '...' +connection_config.postgresql.dbname = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +Similarly, when using a PostgreSQL instance with Google CloudSQL +([quickstart](https://cloud.google.com/sql/docs/postgres/quickstart), +[connect-overview](https://cloud.google.com/sql/docs/postgres/connect-overview)), +one could also use SSL option if applicable. + +```python +connection_config.postgresql.ssloption.sslmode = '...' # disable, allow, verify-ca, verify-full, etc. +connection_config.postgresql.ssloption.sslcert = '...' +connection_config.postgresql.ssloption.sslkey = '...' +connection_config.postgresql.ssloption.sslpassword = '...' +connection_config.postgresql.ssloption.sslrootcert = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +## Data model + +The Metadata Store uses the following data model to record and retrieve metadata +from the storage backend. + +* `ArtifactType` describes an artifact's type and its properties that are + stored in the metadata store. You can register these types on-the-fly with + the metadata store in code, or you can load them in the store from a + serialized format. Once you register a type, its definition is available + throughout the lifetime of the store. +* An `Artifact` describes a specific instance of an `ArtifactType`, and its + properties that are written to the metadata store. +* An `ExecutionType` describes a type of component or step in a workflow, and + its runtime parameters. +* An `Execution` is a record of a component run or a step in an ML workflow + and the runtime parameters. An execution can be thought of as an instance of + an `ExecutionType`. Executions are recorded when you run an ML pipeline or + step. +* An `Event` is a record of the relationship between artifacts and executions. + When an execution happens, events record every artifact that was used by the + execution, and every artifact that was produced. These records allow for + lineage tracking throughout a workflow. By looking at all events, MLMD knows + what executions happened and what artifacts were created as a result. MLMD + can then recurse back from any artifact to all of its upstream inputs. +* A `ContextType` describes a type of conceptual group of artifacts and + executions in a workflow, and its structural properties. For example: + projects, pipeline runs, experiments, owners etc. +* A `Context` is an instance of a `ContextType`. It captures the shared + information within the group. For example: project name, changelist commit + id, experiment annotations etc. It has a user-defined unique name within its + `ContextType`. +* An `Attribution` is a record of the relationship between artifacts and + contexts. +* An `Association` is a record of the relationship between executions and + contexts. + +## MLMD Functionality + +Tracking the inputs and outputs of all components/steps in an ML workflow and +their lineage allows ML platforms to enable several important features. The +following list provides a non-exhaustive overview of some of the major benefits. + +* **List all Artifacts of a specific type.** Example: all Models that have + been trained. +* **Load two Artifacts of the same type for comparison.** Example: compare + results from two experiments. +* **Show a DAG of all related executions and their input and output artifacts + of a context.** Example: visualize the workflow of an experiment for + debugging and discovery. +* **Recurse back through all events to see how an artifact was created.** + Examples: see what data went into a model; enforce data retention plans. +* **Identify all artifacts that were created using a given artifact.** + Examples: see all Models trained from a specific dataset; mark models based + upon bad data. +* **Determine if an execution has been run on the same inputs before.** + Example: determine whether a component/step has already completed the same + work and the previous output can just be reused. +* **Record and query context of workflow runs.** Examples: track the owner and + changelist used for a workflow run; group the lineage by experiments; manage + artifacts by projects. +* **Declarative nodes filtering capabilities on properties and 1-hop + neighborhood nodes.** Examples: look for artifacts of a type and under some + pipeline context; return typed artifacts where a given property’s value is + within a range; find previous executions in a context with the same inputs. + +See the +[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) for +an example that shows you how to use the MLMD API and the metadata store to +retrieve lineage information. + +### Integrate ML Metadata into your ML Workflows + +If you are a platform developer interested in integrating MLMD into your system, +use the example workflow below to use the low-level MLMD APIs to track the +execution of a training task. You can also use higher-level Python APIs in +notebook environments to record experiment metadata. + +![ML Metadata Example Flow](images/mlmd_flow.png) + +1) Register artifact types + +```python +# Create ArtifactTypes, e.g., Data and Model +data_type = metadata_store_pb2.ArtifactType() +data_type.name = "DataSet" +data_type.properties["day"] = metadata_store_pb2.INT +data_type.properties["split"] = metadata_store_pb2.STRING +data_type_id = store.put_artifact_type(data_type) + +model_type = metadata_store_pb2.ArtifactType() +model_type.name = "SavedModel" +model_type.properties["version"] = metadata_store_pb2.INT +model_type.properties["name"] = metadata_store_pb2.STRING +model_type_id = store.put_artifact_type(model_type) + +# Query all registered Artifact types. +artifact_types = store.get_artifact_types() +``` + +2) Register execution types for all steps in the ML workflow + +```python +# Create an ExecutionType, e.g., Trainer +trainer_type = metadata_store_pb2.ExecutionType() +trainer_type.name = "Trainer" +trainer_type.properties["state"] = metadata_store_pb2.STRING +trainer_type_id = store.put_execution_type(trainer_type) + +# Query a registered Execution type with the returned id +[registered_type] = store.get_execution_types_by_id([trainer_type_id]) +``` + +3) Create an artifact of DataSet ArtifactType + +```python +# Create an input artifact of type DataSet +data_artifact = metadata_store_pb2.Artifact() +data_artifact.uri = 'path/to/data' +data_artifact.properties["day"].int_value = 1 +data_artifact.properties["split"].string_value = 'train' +data_artifact.type_id = data_type_id +[data_artifact_id] = store.put_artifacts([data_artifact]) + +# Query all registered Artifacts +artifacts = store.get_artifacts() + +# Plus, there are many ways to query the same Artifact +[stored_data_artifact] = store.get_artifacts_by_id([data_artifact_id]) +artifacts_with_uri = store.get_artifacts_by_uri(data_artifact.uri) +artifacts_with_conditions = store.get_artifacts( + list_options=mlmd.ListOptions( + filter_query='uri LIKE "%/data" AND properties.day.int_value > 0')) +``` + +4) Create an execution of the Trainer run + +```python +# Register the Execution of a Trainer run +trainer_run = metadata_store_pb2.Execution() +trainer_run.type_id = trainer_type_id +trainer_run.properties["state"].string_value = "RUNNING" +[run_id] = store.put_executions([trainer_run]) + +# Query all registered Execution +executions = store.get_executions_by_id([run_id]) +# Similarly, the same execution can be queried with conditions. +executions_with_conditions = store.get_executions( + list_options = mlmd.ListOptions( + filter_query='type = "Trainer" AND properties.state.string_value IS NOT NULL')) +``` + +5) Define the input event and read data + +```python +# Define the input event +input_event = metadata_store_pb2.Event() +input_event.artifact_id = data_artifact_id +input_event.execution_id = run_id +input_event.type = metadata_store_pb2.Event.DECLARED_INPUT + +# Record the input event in the metadata store +store.put_events([input_event]) +``` + +6) Declare the output artifact + +```python +# Declare the output artifact of type SavedModel +model_artifact = metadata_store_pb2.Artifact() +model_artifact.uri = 'path/to/model/file' +model_artifact.properties["version"].int_value = 1 +model_artifact.properties["name"].string_value = 'MNIST-v1' +model_artifact.type_id = model_type_id +[model_artifact_id] = store.put_artifacts([model_artifact]) +``` + +7) Record the output event + +```python +# Declare the output event +output_event = metadata_store_pb2.Event() +output_event.artifact_id = model_artifact_id +output_event.execution_id = run_id +output_event.type = metadata_store_pb2.Event.DECLARED_OUTPUT + +# Submit output event to the Metadata Store +store.put_events([output_event]) +``` + +8) Mark the execution as completed + +```python +trainer_run.id = run_id +trainer_run.properties["state"].string_value = "COMPLETED" +store.put_executions([trainer_run]) +``` + +9) Group artifacts and executions under a context using attributions and +assertions artifacts + +```python +# Create a ContextType, e.g., Experiment with a note property +experiment_type = metadata_store_pb2.ContextType() +experiment_type.name = "Experiment" +experiment_type.properties["note"] = metadata_store_pb2.STRING +experiment_type_id = store.put_context_type(experiment_type) + +# Group the model and the trainer run to an experiment. +my_experiment = metadata_store_pb2.Context() +my_experiment.type_id = experiment_type_id +# Give the experiment a name +my_experiment.name = "exp1" +my_experiment.properties["note"].string_value = "My first experiment." +[experiment_id] = store.put_contexts([my_experiment]) + +attribution = metadata_store_pb2.Attribution() +attribution.artifact_id = model_artifact_id +attribution.context_id = experiment_id + +association = metadata_store_pb2.Association() +association.execution_id = run_id +association.context_id = experiment_id + +store.put_attributions_and_associations([attribution], [association]) + +# Query the Artifacts and Executions that are linked to the Context. +experiment_artifacts = store.get_artifacts_by_context(experiment_id) +experiment_executions = store.get_executions_by_context(experiment_id) + +# You can also use neighborhood queries to fetch these artifacts and executions +# with conditions. +experiment_artifacts_with_conditions = store.get_artifacts( + list_options = mlmd.ListOptions( + filter_query=('contexts_a.type = "Experiment" AND contexts_a.name = "exp1"'))) +experiment_executions_with_conditions = store.get_executions( + list_options = mlmd.ListOptions( + filter_query=('contexts_a.id = {}'.format(experiment_id)))) +``` + +## Use MLMD with a remote gRPC server + +You can use MLMD with remote gRPC servers as shown below: + +* Start a server + +```bash +bazel run -c opt --define grpc_no_ares=true //ml_metadata/metadata_store:metadata_store_server +``` + +By default, the server uses a fake in-memory db per request and does not persist +the metadata across calls. It can also be configured with a MLMD +`MetadataStoreServerConfig` to use SQLite files or MySQL instances. The config +can be stored in a text protobuf file and passed to the binary with +`--metadata_store_server_config_file=path_to_the_config_file`. + +An example `MetadataStoreServerConfig` file in text protobuf format: + +```textpb +connection_config { + sqlite { + filename_uri: '/tmp/test_db' + connection_mode: READWRITE_OPENCREATE + } +} +``` + +* Create the client stub and use it in Python + +```python +from grpc import insecure_channel +from ml_metadata.proto import metadata_store_pb2 +from ml_metadata.proto import metadata_store_service_pb2 +from ml_metadata.proto import metadata_store_service_pb2_grpc + +channel = insecure_channel('localhost:8080') +stub = metadata_store_service_pb2_grpc.MetadataStoreServiceStub(channel) +``` + +* Use MLMD with RPC calls + +```python +# Create ArtifactTypes, e.g., Data and Model +data_type = metadata_store_pb2.ArtifactType() +data_type.name = "DataSet" +data_type.properties["day"] = metadata_store_pb2.INT +data_type.properties["split"] = metadata_store_pb2.STRING + +request = metadata_store_service_pb2.PutArtifactTypeRequest() +request.all_fields_match = True +request.artifact_type.CopyFrom(data_type) +stub.PutArtifactType(request) + +model_type = metadata_store_pb2.ArtifactType() +model_type.name = "SavedModel" +model_type.properties["version"] = metadata_store_pb2.INT +model_type.properties["name"] = metadata_store_pb2.STRING + +request.artifact_type.CopyFrom(model_type) +stub.PutArtifactType(request) +``` + +## Upgrade the MLMD library + +When using a new MLMD release or your own build with an existing MLMD database, +there may be changes to the database schema. Unless a breaking change is +explicitly mentioned in the release note, all MLMD database schema changes are +transparent for the MLMD API users. If there is a breaking change notice, then +old databases can still be upgraded to use the new MLMD library. + +When the MLMD library connects to the database, it compares the expected schema +version of the MLMD library (`library_version`) with the schema version +(`db_version`) recorded in the given database. By default, MLMD will check the +compatibility and raise errors when the versions are incompatible. + +* If `library_version` is compatible with `db_version`, nothing happens. +* If `library_version` is newer than `db_version`, and auto-migration is not + enabled, then MLMD raises a failed precondition error with the following + message: + + ``` + MLMD database version $db_version is older than library version + $library_version. Schema migration is disabled. Please upgrade the + database then use the library version; or switch to a older library + version to use the current database. + ``` + +* If `library_version` is older than `db_version`, by default MLMD library + returns errors to prevent any data loss. In this case, you should upgrade + the library version before using that database. + +### Upgrade the database schema + +MLMD provides utilities to upgrade the database version. + +For example, when connecting to a backend with a Python library: + +```python {highlight="range:enable,True"} +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +store = metadata_store.MetadataStore(connection_config, + enable_upgrade_migration=True) +``` + +Or when using gRPC server, set the MetadataStoreServerConfig as follows: + +```python +connection_config { + ... +} +migration_options { + enable_upgrade_migration: true +} +``` + +MLMD then evolves the database by executing a series of migration scripts. If +the backend supports DDL queries within a transaction (e.g., SQLite), MLMD runs +the steps together within a single transaction, and the transaction is +rolled-back when an error occurs. The migration script is provided together with +any schema-change commit and verified through testing. + +Note: The migration DDLs in MySQL are not transactional. When using MySQL, there +should only be a single connection with the upgrade migration enabled to use the +old database. Take a backup of the database before upgrading to prevent +potential data losses. + +### Downgrade the database schema + +A misconfiguration in the deployment of MLMD may cause an accidental upgrade, +e.g., when you tries out a new version of the library and accidentally connect +to the production instance of MLMD and upgrade the database. To recover from +these situations, MLMD provides a downgrade feature. During connection, if the +migration options specify the `downgrade_to_schema_version`, MLMD will run a +downgrade transaction to revert the schema version and migrate the data, then +terminate the connection. Once the downgrade is done, use the older version of +the library to connect to the database. + +For example: + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +metadata_store.downgrade_schema(connection_config, + downgrade_to_schema_version = 0) +``` + +Note: When downgrading, MLMD prevents data loss as much as possible. However, +newer schema versions might be inherently more expressive and hence a downgrade +can introduce data loss. When using backends that do not support DDL +transactions (e.g., MySQL), the database should be backed up before downgrading +and the downgrade script should be the only MLMD connection to the database. + +The list of `schema_version` used in MLMD releases are: + +ml-metadata (MLMD) | schema_version +------------------ | -------------- +1.16.0 | 10 +1.15.0 | 10 +1.14.0 | 10 +1.13.1 | 10 +1.13.0 | 10 +1.12.0 | 10 +1.11.0 | 10 +1.10.0 | 8 +1.9.0 | 8 +1.8.0 | 8 +1.7.0 | 8 +1.6.0 | 7 +1.5.0 | 7 +1.4.0 | 7 +1.3.0 | 7 +1.2.0 | 7 +1.1.0 | 7 +1.0.0 | 6 +0.30.0 | 6 +0.29.0 | 6 +0.28.0 | 6 +0.27.0 | 6 +0.26.0 | 6 +0.25.1 | 6 +0.24.0 | 5 +0.23.0 | 5 +0.22.1 | 5 +0.21.2 | 4 +0.15.2 | 4 +0.14.0 | 4 +0.13.2 | 0 + +## Resources + +The MLMD library has a high-level API that you can readily use with your ML +pipelines. See the +[MLMD API documentation](https://www.tensorflow.org/tfx/ml_metadata/api_docs/python/mlmd) +for more details. + +Check out +[MLMD Declarative Nodes Filtering](https://github.com/google/ml-metadata/blob/v1.2.0/ml_metadata/proto/metadata_store.proto#L708-L786) +to learn how to use MLMD declarative nodes filtering capabilities on properties +and 1-hop neighborhood nodes. + +Also check out the +[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) to +learn how to use MLMD to trace the lineage of your pipeline components. diff --git a/mkdocs.yml b/mkdocs.yml index 2cf697934..6daf47ca6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,15 +107,15 @@ extra_javascript: watch: - ml_metadata nav: - - Getting Started: get_started.md + - Guide: index.md - Tutorial: https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/ - API: - mlmd: - - Overview: api/mlmd/root.md + - Overview: api/mlmd/ - mlmd: api/mlmd/mlmd.md - mlmd.errors: - - Overview: api/mlmd.errors/root.md + - Overview: api/mlmd.errors/ - mlmd.errors: api/mlmd.errors/mlmd.errors.md - mlmd.proto: - - Overview: api/mlmd.proto/root.md + - Overview: api/mlmd.proto/ - mlmd.proto: api/mlmd.proto/mlmd.proto.md From f5de8a54b4dc9765e7e62fdfaefbf7bbeff0e9c3 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:03:40 -0700 Subject: [PATCH 21/31] Change links to point internally --- g3doc/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/g3doc/index.md b/g3doc/index.md index f9908a2f0..37f11e6f1 100644 --- a/g3doc/index.md +++ b/g3doc/index.md @@ -190,7 +190,7 @@ following list provides a non-exhaustive overview of some of the major benefits. within a range; find previous executions in a context with the same inputs. See the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) for +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) for an example that shows you how to use the MLMD API and the metadata store to retrieve lineage information. @@ -555,7 +555,7 @@ ml-metadata (MLMD) | schema_version The MLMD library has a high-level API that you can readily use with your ML pipelines. See the -[MLMD API documentation](https://www.tensorflow.org/tfx/ml_metadata/api_docs/python/mlmd) +[MLMD API documentation](../api/mlmd) for more details. Check out @@ -564,5 +564,5 @@ to learn how to use MLMD declarative nodes filtering capabilities on properties and 1-hop neighborhood nodes. Also check out the -[MLMD tutorial](https://www.tensorflow.org/tfx/tutorials/mlmd/mlmd_tutorial) to +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) to learn how to use MLMD to trace the lineage of your pipeline components. From 61b75e9324cf9a5d742141ad54a235358f213f69 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:25:57 -0700 Subject: [PATCH 22/31] Remove logo --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 6daf47ca6..399112061 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,7 +32,6 @@ theme: toggle: icon: material/brightness-4 name: Switch to system preference - logo: images/tf_full_color_primary_icon.svg favicon: images/favicon.png features: From 946ef6098407ba5ca3a4b501bcc47a479cd5b915 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:32:52 -0700 Subject: [PATCH 23/31] Fix broken link --- g3doc/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/g3doc/index.md b/g3doc/index.md index 37f11e6f1..5cdf9b9ae 100644 --- a/g3doc/index.md +++ b/g3doc/index.md @@ -555,7 +555,7 @@ ml-metadata (MLMD) | schema_version The MLMD library has a high-level API that you can readily use with your ML pipelines. See the -[MLMD API documentation](../api/mlmd) +[MLMD API documentation](api/mlmd.md) for more details. Check out From 50ed6c1af802f335dc46bee0d1aa989dbbbdeac9 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 17:22:07 -0700 Subject: [PATCH 24/31] Fix admonitions --- g3doc/index.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/g3doc/index.md b/g3doc/index.md index 5cdf9b9ae..3b2904b37 100644 --- a/g3doc/index.md +++ b/g3doc/index.md @@ -484,10 +484,11 @@ the steps together within a single transaction, and the transaction is rolled-back when an error occurs. The migration script is provided together with any schema-change commit and verified through testing. -Note: The migration DDLs in MySQL are not transactional. When using MySQL, there -should only be a single connection with the upgrade migration enabled to use the -old database. Take a backup of the database before upgrading to prevent -potential data losses. +!!! Note + The migration DDLs in MySQL are not transactional. When using MySQL, there + should only be a single connection with the upgrade migration enabled to use the + old database. Take a backup of the database before upgrading to prevent + potential data losses. ### Downgrade the database schema @@ -509,11 +510,12 @@ metadata_store.downgrade_schema(connection_config, downgrade_to_schema_version = 0) ``` -Note: When downgrading, MLMD prevents data loss as much as possible. However, -newer schema versions might be inherently more expressive and hence a downgrade -can introduce data loss. When using backends that do not support DDL -transactions (e.g., MySQL), the database should be backed up before downgrading -and the downgrade script should be the only MLMD connection to the database. +!!! Note + When downgrading, MLMD prevents data loss as much as possible. However, + newer schema versions might be inherently more expressive and hence a downgrade + can introduce data loss. When using backends that do not support DDL + transactions (e.g., MySQL), the database should be backed up before downgrading + and the downgrade script should be the only MLMD connection to the database. The list of `schema_version` used in MLMD releases are: From 384405721cf741d35acfa5aeac135fcf8024558d Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 23 Sep 2024 17:27:07 -0700 Subject: [PATCH 25/31] Add white background to png files --- .github/workflows/cd-docs.yml | 2 +- g3doc/stylesheets/extra.css | 4 ++++ requirements-docs.txt | 8 ++++++++ setup.py | 16 +++++++--------- 4 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 requirements-docs.txt diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index 9a211fb34..36a0cb949 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -39,7 +39,7 @@ jobs: mkdocs-material- - name: Install Dependencies - run: pip install mkdocs mkdocs-material mkdocstrings[python] griffe-inherited-docstrings mkdocs-autorefs black mkdocs-jupyter mkdocs-caption + run: pip install -r requirements-docs.txt - name: Deploy to GitHub Pages run: mkdocs gh-deploy --force diff --git a/g3doc/stylesheets/extra.css b/g3doc/stylesheets/extra.css index e734efefd..beaf9694e 100644 --- a/g3doc/stylesheets/extra.css +++ b/g3doc/stylesheets/extra.css @@ -13,3 +13,7 @@ width: 100%; aspect-ratio: 16 / 9; } + +p img{ + background: white; +} diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 000000000..2c76c4f30 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,8 @@ +mkdocs +mkdocs-material +mkdocstrings[python] +griffe-inherited-docstrings +mkdocs-autorefs +black +mkdocs-jupyter +mkdocs-caption diff --git a/setup.py b/setup.py index 55a642216..0bb2f9f0d 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,11 @@ def run(self): with open('README.md') as fp: _LONG_DESCRIPTION = fp.read() +# Get documentation build requirements +with open("requirements-docs.txt", "r") as fp: + docs_reqs = fp.readlines() +docs_reqs = [req.replace("\n", "") for req in docs_reqs] + setup( name='ml-metadata', version=__version__, @@ -167,15 +172,8 @@ def run(self): extras_require={ 'lint': ['pre-commit'], # TODO: Pin versions for docs - "docs": ["mkdocs", - "mkdocs-material", - "mkdocstrings[python]", - "griffe-inherited-docstrings", - "mkdocs-autorefs", - "black", - "mkdocs-caption" - ] - } + "docs": docs_reqs + }, python_requires='>=3.9,<4', packages=find_packages(), include_package_data=True, From a3adca30f5027f9a2faa2c31e8d5c756b0baea83 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Sat, 28 Sep 2024 23:34:04 -0700 Subject: [PATCH 26/31] Add build docs check on pull request --- .github/workflows/cd-docs.yml | 25 +- .gitignore | 4 + g3doc/api/mlmd.errors/index.md | 45 -- g3doc/api/mlmd.errors/mlmd.errors.md | 5 - g3doc/api/mlmd.proto/index.md | 35 -- g3doc/api/mlmd.proto/mlmd.proto.md | 5 - g3doc/api/mlmd/index.md | 23 - g3doc/api/mlmd/mlmd.md | 5 - g3doc/images/favicon.png | Bin 404 -> 0 bytes g3doc/images/mlmd_flow.png | Bin 50049 -> 0 bytes g3doc/images/mlmd_overview.png | Bin 55112 -> 0 bytes g3doc/images/tf_full_color_primary_icon.svg | 1 - g3doc/index.md | 570 -------------------- g3doc/javascripts/mathjax.js | 19 - g3doc/stylesheets/extra.css | 19 - mkdocs.yml | 8 +- 16 files changed, 21 insertions(+), 743 deletions(-) delete mode 100644 g3doc/api/mlmd.errors/index.md delete mode 100644 g3doc/api/mlmd.errors/mlmd.errors.md delete mode 100644 g3doc/api/mlmd.proto/index.md delete mode 100644 g3doc/api/mlmd.proto/mlmd.proto.md delete mode 100644 g3doc/api/mlmd/index.md delete mode 100644 g3doc/api/mlmd/mlmd.md delete mode 100644 g3doc/images/favicon.png delete mode 100644 g3doc/images/mlmd_flow.png delete mode 100644 g3doc/images/mlmd_overview.png delete mode 100644 g3doc/images/tf_full_color_primary_icon.svg delete mode 100644 g3doc/index.md delete mode 100644 g3doc/javascripts/mathjax.js delete mode 100644 g3doc/stylesheets/extra.css diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index 36a0cb949..eaeb2a2bc 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -2,9 +2,9 @@ name: deploy-docs on: workflow_dispatch: push: - # Uncomment these lines before merge - # branches: - # - master + branches: + - master + pull_request: permissions: contents: write jobs: @@ -14,15 +14,10 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Configure Git Credentials - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' cache: 'pip' cache-dependency-path: | setup.py @@ -42,4 +37,12 @@ jobs: run: pip install -r requirements-docs.txt - name: Deploy to GitHub Pages - run: mkdocs gh-deploy --force + if: (github.event_name != 'pull_request') + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + mkdocs gh-deploy --force + + - name: Build docs to check for errors + run: mkdocs build --verbose + if: (github.event_name == 'pull_request') diff --git a/.gitignore b/.gitignore index ed7d69318..63e111c79 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Bazel build files +bazel-* + diff --git a/g3doc/api/mlmd.errors/index.md b/g3doc/api/mlmd.errors/index.md deleted file mode 100644 index 04562c7fa..000000000 --- a/g3doc/api/mlmd.errors/index.md +++ /dev/null @@ -1,45 +0,0 @@ -# mlmd.errors - -Exception types for MLMD errors. - -## Classes - -[`class AbortedError`][ml_metadata.errors.AbortedError]: The operation was aborted, typically due to a concurrent action. - -[`class AlreadyExistsError`][ml_metadata.errors.AlreadyExistsError]: Raised when an entity that we attempted to create already exists. - -[`class CancelledError`][ml_metadata.errors.CancelledError]: Raised when an operation or step is cancelled. - -[`class DataLossError`][ml_metadata.errors.DataLossError]: Raised when unrecoverable data loss or corruption is encountered. - -[`class DeadlineExceededError`][ml_metadata.errors.DeadlineExceededError]: Raised when a deadline expires before an operation could complete. - -[`class FailedPreconditionError`][ml_metadata.errors.FailedPreconditionError]: Raised when the system is not in a state to execute an operation. - -[`class InternalError`][ml_metadata.errors.InternalError]: Raised when the system experiences an internal error. - -[`class InvalidArgumentError`][ml_metadata.errors.InvalidArgumentError]: Raised when an operation receives an invalid argument. - -[`class NotFoundError`][ml_metadata.errors.NotFoundError]: Raised when a requested entity was not found. - -[`class OutOfRangeError`][ml_metadata.errors.OutOfRangeError]: Raised when an operation iterates past the valid input range. - -[`class PermissionDeniedError`][ml_metadata.errors.PermissionDeniedError]: Raised when the caller does not have permission to run an operation. - -[`class ResourceExhaustedError`][ml_metadata.errors.ResourceExhaustedError]: Some resource has been exhausted. - -[`class StatusError`][ml_metadata.errors.StatusError]: A general error class that cast maps Status to typed errors. - -[`class UnauthenticatedError`][ml_metadata.errors.UnauthenticatedError]: The request does not have valid authentication credentials. - -[`class UnavailableError`][ml_metadata.errors.UnavailableError]: Raised when the runtime is currently unavailable. - -[`class UnimplementedError`][ml_metadata.errors.UnimplementedError]: Raised when an operation has not been implemented. - -[`class UnknownError`][ml_metadata.errors.UnknownError]: Raised when an operation failed reason is unknown. - -## Functions - -[`exception_type_from_error_code(...)`][ml_metadata.errors.exception_type_from_error_code]: Returns error class w.r.t. the error_code. - -[`make_exception(...)`][ml_metadata.errors.make_exception]: Makes an exception with the MLMD error code. diff --git a/g3doc/api/mlmd.errors/mlmd.errors.md b/g3doc/api/mlmd.errors/mlmd.errors.md deleted file mode 100644 index 3c1dada6c..000000000 --- a/g3doc/api/mlmd.errors/mlmd.errors.md +++ /dev/null @@ -1,5 +0,0 @@ -# mlmd.errors - -::: ml_metadata.errors - options: - show_if_no_docstring: true diff --git a/g3doc/api/mlmd.proto/index.md b/g3doc/api/mlmd.proto/index.md deleted file mode 100644 index a7cf83f20..000000000 --- a/g3doc/api/mlmd.proto/index.md +++ /dev/null @@ -1,35 +0,0 @@ -# mlmd.proto - -ML Metadata proto module. - -## Classes - -[`class Artifact`][ml_metadata.proto.Artifact]: An artifact represents an input or an output of individual steps in a ML workflow, e.g., a trained model, an input dataset, and evaluation metrics. - -[`class ArtifactType`][ml_metadata.proto.ArtifactType]: A user defined type about a collection of artifacts and their properties that are stored in the metadata store. - -[`class Association`][ml_metadata.proto.Association]: An association represents the relationship between executions and contexts. - -[`class Attribution`][ml_metadata.proto.Attribution]: An attribution represents the relationship between artifacts and contexts. - -[`class ConnectionConfig`][ml_metadata.proto.ConnectionConfig]: A connection configuration specifying the persistent backend to be used with MLMD. - -[`class Context`][ml_metadata.proto.Context]: A context defines a group of artifacts and/or executions. - -[`class ContextType`][ml_metadata.proto.ContextType]: A user defined type about a collection of contexts and their properties that are stored in the metadata store. - -[`class Event`][ml_metadata.proto.Event]: An event records the relationship between artifacts and executions. - -[`class Execution`][ml_metadata.proto.Execution]: An execution describes a component run or a step in an ML workflow along with its runtime parameters, e.g., a Trainer run, a data transformation step. - -[`class ExecutionType`][ml_metadata.proto.ExecutionType]: A user defined type about a collection of executions and their properties that are stored in the metadata store. - -[`class FakeDatabaseConfig`][ml_metadata.proto.FakeDatabaseConfig]: An in-memory database configuration for testing purpose. - -[`class MetadataStoreClientConfig`][ml_metadata.proto.MetadataStoreClientConfig]: A connection configuration to use a MLMD server as the persistent backend. - -[`class MySQLDatabaseConfig`][ml_metadata.proto.MySQLDatabaseConfig]: A connection configuration to use a MySQL db instance as a MLMD backend. - -[`class ParentContext`][ml_metadata.proto.ParentContext]: A parental context represents the relationship between contexts. - -[`class SqliteMetadataSourceConfig`][ml_metadata.proto.SqliteMetadataSourceConfig]: A connection configuration to use a Sqlite db file as a MLMD backend. diff --git a/g3doc/api/mlmd.proto/mlmd.proto.md b/g3doc/api/mlmd.proto/mlmd.proto.md deleted file mode 100644 index e190a876c..000000000 --- a/g3doc/api/mlmd.proto/mlmd.proto.md +++ /dev/null @@ -1,5 +0,0 @@ -# mlmd.proto - -::: ml_metadata.proto - options: - show_if_no_docstring: true diff --git a/g3doc/api/mlmd/index.md b/g3doc/api/mlmd/index.md deleted file mode 100644 index 4a1890b72..000000000 --- a/g3doc/api/mlmd/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# mlmd - -Init module for ML Metadata. - -## Modules - -[`errors`][ml_metadata.errors] module: Exception types for MLMD errors. - -[`proto`][ml_metadata.proto] module: ML Metadata proto module. - -## Classes - -[`class ListOptions`][ml_metadata.ListOptions]: Defines the available options when listing nodes. - -[`class MetadataStore`][ml_metadata.MetadataStore]: A store for the metadata. - -[`class OrderByField`][ml_metadata.OrderByField]: Defines the available fields to order results in ListOperations. - -## Functions - -[`downgrade_schema(...)`][ml_metadata.downgrade_schema]: Downgrades the db specified in the connection config to a schema version. - - diff --git a/g3doc/api/mlmd/mlmd.md b/g3doc/api/mlmd/mlmd.md deleted file mode 100644 index 7ac8b930f..000000000 --- a/g3doc/api/mlmd/mlmd.md +++ /dev/null @@ -1,5 +0,0 @@ -# mlmd - -::: ml_metadata - options: - show_submodules: false diff --git a/g3doc/images/favicon.png b/g3doc/images/favicon.png deleted file mode 100644 index 00a8af6dabc6bd2705678240ab4a59398f490e80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 404 zcmV;F0c-w=P)bWUp}xZee%_FF+E28Gr&LzQXejWB zFF+E26#x_I#XCg22?r-%Yj0PYEp7nRKcP5}NPfXW>d zAQS*X4L}sYk;G^K2?3k{u>_C=$Zl&HU|oxb9nw&M7hvPP797D5^YtB!^1A>lE_VEA zWtT8;3G)qN)hY)JtNaxtw(!6H4J$lDHv|41;8=iuIsgd)i~wGMGznl`3oC$`4v;qk yX22l>+=$W4)dl4JFr1FvuXH>&TgG!!vezfc^{4$!^Uogu0000>?2^c<1p0!RxuE z`|7V_j;k6a^hm__csy=6pRYJl=J3ut=^MYJizG|xx3U&U9et#5W2g>Q`?S$i84u`V zXiuUpvw%V4bscVK-Ehuz*290CxOR*}?2fAmSYz%ZxnYTEkxy4cgXm&iE@IsaxlqMbvzN3-L#jHWFs z&grc`1Zv*KOof`%ZjQ2@dea31~F0>tl z`zT<7!M+%0$KO}O>~p@g8v{T43WGuF>pxyw@&4mh-f@k<<$G$g)r#sr>lg*B2-dy> zhYBjbJ^uMQCfM7Fq{FLmFnGx<1oSfbTxs-2D9@d(l?7k520QBWD`M>8JF86%frAU_Z7YL;?mwv1toBgQy?sSWTm3yfe6sk} zZ(M)aEWI;}avBPpsXV!Htak@HO@t%Mg zU?k*cg!mg7N4Q%}zdFesjNhWVM{4aXXz>nh zepil+w+>z#Bl*mR>})h`620Q|SI{I@IBr~;O(~`?h7Rg)-EB~^jUU_?TzL>F9-Qct z9=dT!ODt=p`4USl?j+SJ6f;*2X$ubO<$cgnfLd)37hAUsY7iMbd-R`X?0k3Mc38d- zsa=)gj9+Oj!vDkre&;yVmvyOD62HAv*tz_9XSua_2Xx)~#^U-5eOVQ`yWO6QYBj`! z&+l&My`LGV%J2%Fl*I-tO=f=W4;H$;7XQ&?%_7GOBj?Kef$TWe5@${{z}2-$gQZv< ze1}lJK60yddelPL`j8_mZlGkNWkxi^!_>?sg!gvKk9DqVgDCfLXzlxr1!D2RTa)hk z1m08I!F_o0m5>nJEGBB(MiM7A=6w(2NnaLxy&Hu3S8^ zjkmiYVp)esvbD?!yp)ma6OxObSQP^GL8t?>!7E+7e%k?)?oH(Sa{u+MZ(6L=?iL}X z0f-C%7v;FL72g>xzA~Fx>PGuc`#4IHwDl%1w|@n`PRjB!8`yItovoT#lw2=?;9$$P z-d$Aim_w|`_5r3pxVTp!ax3{q+P3gpxr%DHs}{0f^MF5J;tXxU69Gu~8-xy4ZOu?K z5=;6Z(b`uOp~(p`Op?kcqViuEXt90KC@4-M>+W-CZNrtzJR>ljpHzq&)R}y5Df@a# z@s$5c4U!XED^k^YXmeatpF$bRwAm29)waHIV=EN%yn!q-wA$!e6FyMJ8p_;1$^H!c zgj~lE02e@t8_!B`F7|98HC-0G#n)P-3R8XRl=>j;jk+5?AWF>XW#9Cbtv|4(gA&Wi zGJ=Xn6bsEuC0Azqw{;hN{YkKdxUJwG$LA;`XUnu$lqCC+R#fd|Q}SY|{xbAK*lu`5 zplu(^TP+=B#-;A98S`4{11`lK|Uq67cRkQi0aoTMwQ=KK6#%3@mgHfRb*>`4n8)^N1s@0%vBNBT(%T{xfr!b`mUvXuy;yR zZ&m^>5NC&Q)UF|0B^}z>-*&_%1G!w8CF{T5);%{JSu~-f>HMzVVtwUy$NJGGysdN( zLIcD!>LReCoR8RayuZCYzunOnzuiQKDxVxI5`Sn|-D}yy~%iap>pzt3KAub(D za2%tLlb*#Fbt(-+5Uj1nA!k|9QCeuyP-5`n+yFm{U$c<In@`a7a@k$LR;S3 z&dPdI7e^z0x=0FsPDn##nk)0gX`3Q^yd%z0czH`2nbr!fgAO@xED8-&%fIaNjyTMY z9_wprG5cxC!yyTs4@>Ih?sn8Fi-O5dN9I+#5u6jsKEF?9aHVwkhLySD>d31N7L~Q) zJIZPn;01G1vYkUdrVWS8KRk1I3Hp5YMQ$fsoV-}66$7odAztLqM8@H-qXYZ=>XV~L zT|NxTB5-^9(ye*tTJIQprK?Ee8)ov%(>~LSX8P)0c7`N*t?xR8GI=gV*#$-u(qT#} z50UbV5j#yHgrHY$){EZJfCcr*){bGxNS!kA+@O`->W$Hhkao0^S*V*e#Whu;NGZUQ zcyVfGlHPo!5IyBK6w-2ijmmXzv0Tdn+_RE|cs=qm1!Fos+LSqIvjF99-HM_pj+XUz z_^LZpGSLb>`Gl5bJHt(8f%PtBOKdAn+>{Yc=)_xqO$6Q~kKUly>~Z9KCTLHZeoo+S z#zmTUp=U1nNT)rSw6}e@K3|G6!sHHwVdla>j3{QU<=3Iioh|X^km=I=$!LeO?uu{L zpQdhE{}AiNnm#~UF2vMboeg9*9fKHZ$1@|H>zu<*6e(#IIanW+iJc4)RCs1`c3NEL zk0XQRpt7QD@#ke}QN}%vFX<^@;~h;3qGw%N}ov z90szte4}wa>ocTv8-2%uUf=wvzy;{_o?4HzoT!>I@r=zC$-TIF))%8mok=8W+$@v= zmWsDW;5GijBJ3u@p*?=$$+gh2DxWVWP7u;=)M~y95{b08j^p| z-#+_RVe<>0ni->fTHaQ7#{1+jTf@Uz(59RIN+7KwrS*+V+Fk(}t1}LSL){?-UwAE8 zt}m}S%uPv9>d< zNcl3?dkUO!5X1VyjBtzDy!#+nud6L8ck)RyzS+HjJdUh#O1&I(basPlx^!l zI`cZ45eN8ds^bjPasDF|SH$)pHncHRFKNGab;i9*^a&X(wy(w`)R}d*D;+o*r^QS{ z$C^HolsMi`Utz1#^@q9~e7gpn2woZ)3&j-I7;l0~xso*_y4|`13IiK2SnW(Le8T3j z5W45wm)EkA7-cr@AZAKsf$W}N>yFA;ouwP7npj+3-nv80>tv)lTJHzUM#+LL-$@Ty zlXtAA4!dTn=tI za5#X`{{xl(4Tt{|mH(S2{0C|HyFtIq1MJ;b^g#Z9G3fu5UN91X8hG|!<4in~?(<(P z`Zaz3wCJB#|9a5>M1KH^)6-$a`kT1?zfahIH4iYTOz|%U_CFc){|g`czghYJZ6FK; zSvo>6l^85jJs(&2fp2uGpvtXZbJe*ucVuXrIwbj!m2j4Y@Utuu1Ck)}xflnz+Im_x z%UE$h48tUn&rbxz5LvYlDi})A5iuGYZ8Q-gBp*vnVsdWMAGGzfzQ0A*pSfC%8+{EvR{K|d?3EZCU<|4?mk{L5E8Vfu@EH9 zx%YSR#D_7$d>*qKyqSEwkv${j|5-52(MvsM(5~*Y&a)2qzvqehH7GkB!?S~);bE|R(LGaX7x)aQ?%o5@cTRHaoO~%f7#NQ@c&g10Nh zGcB`TW8m^uYaS)(yTtVd9t(2+8Z#?zeSMtvS1elw3&Ry_U1w`60-s&e5eK~&pB2h^ zV=;G{3-83 zF~YuE)55NXB{@FPkx+h2qE+is!PXYW=5Xo5rf-@lPz|*8WGILs2*UlRPrQr7B-;^$ z(4vn+i;0HMidEf8PveIU3=vh|Iz0^i9#%+z)y+c?Cz<(14m?96}9L;Xt1O6 z-`(jqe|Q8$47-W)l;M=OG|d&UAM^+2^WG8t7jvEjv8oqsjd47V+4=bv zR$Bp9X6@W=)7JSBrP)k_z7iXjyUm$(s18Uwq%qv2S#xd_?@}r0bMOUWE}#ma|Mw*3$eV<}f$fT>0u0 ze}MYwowrp+eNWASIxB3o(nw(1o}YB2^MBQb(R}&TZrYRRVcQna0lv;i2nCT3{ONt^ zMW$#5g6a!BeV=jHQmuj&H!)^3NyYu{0K=Bb%LvS!jydHhIs$?CGDF*A=X^c+8mnsF z9kZ*p+s{2hLA;p+9}!(P_sjKgP%}3>=v{9EusUnz}z{Wm>tB*Mux( zBC!1o`)zqc`a)_-zIh?=--{;I?N z>8%X4NTlpYDFqo}WhsG{X`5h3a>!KT@{-v&JS|gjfS^<{uL|$TXjcFpW~@+u)$raQ zix#&S;Y)7Q!fo;FLuw3C$^=>@;?+CxL);N-ITA?-JUrI(wmePrDyw?VQ!eI^Uj zy<72ml(8dq;v-j+Q0Ps6*kt9=f}t0sZueUqZsX#qa}^(x$BhPTV^bp|vpf6-Ty?Kj zVZ8?IPqPjP;tgd3&1OU2om9zy9qPa|8g7^H=4y zi;s_#snY#ayzF+oE#n{tm;`hc>i7?ZC;v^fcBD zAe&IoZ%H$=`2pr)=003im7=(skL&yJ!G=iQNJds3`*99*lngpov*6b_`9qfM%G1;f+ zd&JlKoB|Kvg*85H5e~VPl`0Y18Fqjy#c8|Vo$z2_1xs;jrK1D4CKR;ync1en^kp{Q zOl{#;YTUaahKRteC8CyN6-}$PWZjGAJ#F~*`()-djyHXVH*Pw7KQ6v@m4#uxNprc_ zwX(#D;{&F4yBwXs-~{u2IriBT)??tG6i;!l)d*TH%@9?M|9WwQLl7BEh|AB)l4xx) zybMXr{^Rw@@k9b4y*#FuOisoWMuPfoQ(X}m4AqmK?I5B(LF0#=OpSXG3;*9H zM!rb$bd`gU5OWtt!6CYk?S+g^ocjL`J9ttG|gQ>Hc>LuU=)Y0ZCLT4?^z;T7kOGV$0 zA~u$(q%{)ikNG9Wa5|ILvb0E$TJ(T2*=&XTHwfWAdTxZq?{K1IuzZG4@e%bxwPXfA zUK5T<%wUOkyvB{9f(tv@DQ?2a_803h*rlce{UrY`Zcjh{*pwGtx3d3p@e?BLst<%4YO6->8UyiF%rPD>sCo@r2*GvcgfDf_8g zWurJoRk=Rv&2(DlHmT;14Wq)inuXXXw(rwTEPr!;-TLH`gZ!T?mCrN((9}C{FS2tXoni74bH3CGR-qPj&<-;-IydQRkv2B8hYa3UEwDd^oyjI(I z6Vvi;6yskW+b#$$b@Si|mxyw|+2v*)b1ezPJj~IEQoa%af_2S@JHXtSSi_Y?%_sqI ztAj;K)$t;NlV{B1fK>ymK|r|U2t#^G{Wryqu|cot>P-QEs7kV zs<=*^Kg%1$a296D-0LDYaQuu8ln-N6mxENoqYtA==M_YIf|-66Iq;Ew)QDa|xC?34 zCk*ECxP)kD?L4oWk&4UQ=onJ|T`xiRy2^lr7{_2@iG%U>;NF;fO{;t)7rL+(Pd!}QQiPs?wW(4c6HguApq z^@MF6vkDSKG}P7wTg?+*h%~*e#@h9>#nZ#}txuM?a`33#tGgyHOK9|@;^^pm=@I;1JcaUzrt;-+1EVXo-*cvJrrT>K_ zm4qp8i=p|XH zL9tfz`(&;4plf;xPP|iYNnrW6ohh=O99Z5eSf1CBrg#}~767Y0O^ZJ?P_q6d>t_8{ z^oxWOV{LN&mQ2;R_iL)&X>^HZWH2$g*H%-UH*}16+uc`vJJViV6sq6vX!3%)bY9@Y zmVjv}M*Y;>I9NPE#`s9K?8Tgxp(HOukz=PU)A+xq4yC3VBo}&>Q{0?`242{I-2^## zPHirW7RxB%)<-c_oue{OHe~dUbQk2s)t(XyG_F4B5-VGX8~3?f_^Hx(db9k}5#~LB zv8yd~uCnW5N9_sUd1JF$B0y7E@ZD#gXLBg2QIS2DBvH+{KNw%@miJ%Cmvi$vB6jlp z*{djN_ z9(6o{#x&NI%QOhip|TiDMfH5#Y?;c}=FZ3MCMA|` zTHQB_P|*e>dJ9Ui0`$I2;&%~5l(}@UkJhXod;8Qr{P{q^M!y*AG3sX}O}8+{+>fvl z@M*M+hk22V{HMagvkv+7)LMlC+k^{FAin$FRm~=1Q3Yr>WrBMGBe;s?ij^o; zSo+#zvFlSvyTTg{b}RGBC6%O#<6iSRd~v> z)ii>%xv2NN8Si|Mo`gBiZQliC@MV9Vb@dP*aBq zU_SbLxAOY^-#|Oa{iDK4Pe#&zSP(zXURAF(N+@20&Zv&8>#(QF_z>^HlH8)K^^_YJ z*?Xgz(S0X`N-!#|F(2TVk7-`k_wJ^gv)7RhG^!||$Z{(Js zT<|CT!3&;{_Hx6_@0*bca$ZFF6MX6CRGgNGZ`otbu`G^Rjo`*_2b}WJ zX)o=(sBc#!KjCs-*&#IkgmyJ--2(b`@GB%=Ud60%%|o}2S!G;VIleExw6i1?B|K2v z;%sw&3TCZ zy-=|}nb$CuV9SGU|LQYX;Q=N2>}roi59Yihc6X*tIxIFyri5yRRd;2i5FwlB= zeS2e;mEdqBH5XyKkW9Sg`imsea>?uBB0`&+MRR@D?#-tJ+75r*kye{KXu4R9Y=2EU6J&5f2ZIMjG`@BudW8M`Qfgk=J z&v_f_A5UaCYa=hKx|nEW{+I#frrd98eiRZF%GQJJFa6s-1O0x4e$Cmx znx6j49gKS}y4M6nzdsw1EAZETg6@TTVIXqChQOawbSV*=W%G|WWFsma|JGN3H$xEVb(Mm z=##rx;_|Q0K_P3cw%;4X0L|xD);@aAfd|da!}B1lUeWJy1z37Miy1%%zcL6Ab?fhw z6dm7ZhRimYd~o5#m+eYMyE_9kbqGi|v>*SX;BhqM;A^R9?-*A6bHWO|c_*UQl~2=b2`l`Ozj>xM+w z-KG15DBmxLS<*&dSHRe9oms6d66^azg*$y*-HY4t+~PcP?m@R#TYSYTAzkL1wgtT6 zL&YM%jQ?eCCeVv!Jrr=>%6smkQ0Ug@ogm{GrJ>cG0ARD%#(je#=KwJ)P>-9l-JW_p zE-lbDu<~`KW#;7dUXRU^g!aJ8ezUgyDvQBHJZtF!^RA)4r1^_ZjSa~Pc010)A`K^f zXV~SocoQ^OtAoe5%^X^pjRNGZD{}U$SF?9-2GA!q7HlogVj~5rf4;aop>LBM_u4g8 zbR=4BmdDE`$Qf%zK@Y8dcGa?tuFp{L3HiC4vb(guk?y|awW8%oB3`q4=QX7i9skOp z>${!dk)_HY)>1x%54MU6tHwiXW)bx+A59yRaD$YWkz9ZOV>v2_GUuq zO2$S@xtAhbHF849gZ-B`(Ue^!K;~H3zTV7v1#Bdfx%Tk3mSKmNQS*}*3w3E%kY z@a$ADxMx)n8Fud3O~SR24`-0hhfkx|+@i^*P5bUSlQK3Z0!y2(Zu%tucykyn@4yXC zF)Z={g6=*Ri?ym95uFv3f7zCyrPU9pe_cW}&sBc{$9ZQuoLA2x=UVkzEU&I^!dK(U{6cXWZoa z_6~*LtjYg)=t`sgM*f_Yautl#T2!?u9RAH?DhxA>YKY1!DY(xd?cY4x{|tteexk%* zBzd)m=|oAqRGvdo_RMpCh?!d(Ex_6(!BaCcJl59MN$%Bija}W{3X9uanL5WG9H<|f z8)?c^q`ALee4Pb?Yf$IjgESs&yxNiTHlD<@^-FUVL!T=U#zD32sZ)^gq4O-P*0xp-CFZffG2Yy&za_s2UPPJuPb;2uqy7`RQu zIyV|_3;;UfFlsi?)`u&W>h-lQ%oXVJ0t#%7EE?2wA*8(ZJNil3ylnZvg9_LBvqy?L z<3{M+?AmT&K7PfYX8E=x8&P^r7JfQ*%X0rfV%-N1)`Td%M#N>S!RyR|-W6UX(FCdy zHchjb$K*xf|#)t>X+G^;#LcoO5*rZK@m(i1N>mO(E&X1C8A_)AD{Ay5fq~zvR zM@X+?&r4I=&cQQ3KTUlSJ8tx*V0<|Xc#9GkJoe^ z!lT$$`DLvH3-IfccUxu3Pkj|+P=t+N%ea!6n|sO~1M)BrfyuswaQaCONsQTzK_Nl|Hn(UK`^>2Zw&co{M%dbZ2Wn$s2MUr(5f)M( zo;gh#ZirO;IiIAwg;-u(9DPq#8(QngM1WX2zD_Mu+&f!9r=OUbnI&kaLJY^=@_0vl z0Cs44SP807rF_A#SMow4Flz}ZBz>ywaVXR#m>$xd#Ug@P&EgGG^fv zk}zw5?19aSyHa_d!bon8`;5(=-|64Gv`OK(%^4;itc># z++5_NPVp5N`V_-f+xBG!MTG5)Hpw&{8DcQ}xx{&I=;LxdU?n-ES2d8xOAJy3fod`;7R6VY%GXwzDD$%FMtx&Yas28ipw9QBhBQ6s$Q16Lk|`Cd_nrd zkzsh)OG0H^y5Zf=2kr7@#=#Sb7;c_}=(ZY|LP#7WpHiP-IQsdp;>}jC67z|TO1BIp zR#R&|7Wb4^2O(+mZ0fUCQxdXh`VJxvO8DM5&si+CU}e^ddxPM)E$<*xW8+V0Lw-#) zTNzx%laNi+r-;r(40||G;pyQfyo7V4QlZ2_5X39q0eouBuV0Mks{H}l)BY(dL$ydoqc`X-G;g<3X zp9Hd zHQX-!{=G`xgW8ez&Phb8*ua=+gEqQ{B3Hmu1-51kULqry$*YYEBzdLYr_*!nG!z<& zIn4>r*qU}!^t+Zo;)<3u9F|M$u23{AmP=Ig9KDJ);(PoB9ygXVYTQcO2UPlQ2s_;> z85&z=93Zy)0FXMeO8YxgVQqF z?9Ggb&7vyf-PW-?&&E0w#%t;krDs*H?2sE#qDDB&0vWHt$bLUeLxWXmB~Q>i?xB+8 z_={+vesJs#Tez^Y)F_SvF=9tr!c3K zC6$$k0ja3w#QMGhYAA+-z?jos9zG3EfPuwHDFvtQjUii%i@I-?~x55{HDhzCt*+U@58&MHay zFI8;^y|sM(b@DNB$Hf5|b;~sW02&$eW5itGDl(*ZxdrVH)^EzJsh+znR79$DvW;`c zN{j7+^IseP`!1G>5ngg{G-&zzF0VfjF-ucEeKi@mP?YT!)e`Ug;y~>n#IUgMP`9|g zwWLg7HVX7T|L~W8(!6t0c82CG7nOn`bPiF#U9fNveVQdRNe`G{E%de$T#(ieiMe zWv+9ERqj*1z9_{AW$grr-i0i#MEW?K!DJ16!XW1tChy5_Z)aj#I)u1fLPW1jE!9Gd zTM;#14Vyj(AT!IP(@J~-+)HmT{LOuU4dd5#`NC2uS?pTU$CdgO`^u~b6-nJo6Q#YI zCr(&oIDDQqcV~ILTQqiQDNSE$ccJZ>?;7|QvEJpEf3hhWgWluPdur+)%l#+orB%k? zaY5s9|{fxSNnNXY|ghNB9X6$M`y9e9$((|KR#>O!W6I>l+3Z)@Z zYjYSljK+HpLBZZn0Cd0OEP(UTgf`kqIFk>N;wt;2w-@dGR_9EH{oIeLFE38~ykr~V z_r3SR`5S_4_qXqy*EpM4dEQTwE4_K{>k(TET8-0N|8P=sfACa!u+(97zq_9V7yxC1 zMiXiMCz79r&aAP#D@I1fq>YV@hKZgW zoy_Z>f+l-&+fEv%w=F&5JGt5owB2Czy4^P$-R}XNBha=Bzd8zOhXKy7s>P(TX+6H~ zGw)r@1aT~Z7!mcM7EE`-&oav4oq(xDtow!q0ml*r@#}qs(L!(kiZqeePm^#m>1V=7 zfP_*iodo;zazEoI+AR)npgCdK&^@3hG5l9YB(UI^8>_%DI5avJBj4A3y}oj}jth_t z90~_l8 zWOOf5T<%_uF}M9lJCjqL5a4W?p4LZf*ILY?RsL@CB-u|vec2#HN#}KxN{{PPZ#Yc~ zxus`ji&}as=EuFa>&cesaUY~@m>F_5P%Kg5#?ac_qlq5O+-mdB*c_uszWaB+VBo~- zMBVnTtIJ&A6ul+fbV+MTTc`9$qB1qefxrzR=HO|3rh zLYJ5-;(9jpmRg)9{r-B{r9be!B72VBZUN3?`Oaj@plbf=b~83@xw1>%f$0)c)!(cI z`lxm_#v8ViQNL9!hGUSZfPa2K({CsBYkmjjp4|F8VE6dwuDZ4-$o^?QAY~tUUna96 zUDW8v_d7sVi*f+b_TRtxo56rmf2mIRUmwtP+}?z}R{;tepdB;9%Z@Mo*?;|C-%ORR zbrkm>eSWuDBH>=f7Ab?9CYSmx<=Kh`1O!yjG7Rt+kjO9GbGDoWk%DQ*L$8S&W)>94 z_ItL<^w$c4M^que8P7ejx)*+C)i-$DR{>o0GqZ#3Cfq)>@c;u53;B(>eXfT+-iz}R|@RW)us*_&BfhU~#A zzO#v0=5H`L+4=SKS4ShByF0*!+^;);!H@SZEJFNxSj9e?)a9^t{OjAjH}A=l3;_ld zhQGQ*>V)YvSO|?`1h<;;=AXkF##&9SGpiGCbUKjY>&{=cjXUjkXrA-uz{Ps7r5uE9 z>XohG$Qb?xynBXfr?P9X>-$XIrru)e7DaQ-@BFf#{N{GN7|p`poq1hQvd?lodBAoe z<-qj?R`FT(ZoikPo*n<7ZUyhZ5+>UZ`68w4qUEH*3@+ZovRpMgnf5fi>S= z$J}?{EYeDy)Jid@kbQ!!L*`sp-`A#dxZ}6_aaS^gxs-SM_{K$bf_EGPbUGCygwW0+ zHDno*Aw<=j6{P9-G9Zlvo^YJf;9?fAOh5#o1+jGm)U_nH73m$3QpL8Ar$q}>bx%}Nr4w!qTtNp`T9Nv zlxA#y_WMaq|IQ)xB-oc}b>z(0g&6r=4M~O=Qp%Rm$4A%LrW#;h+ z?R$9a3(Ie~u>z@#W%Gjxpt$CAnW*%s_`K?oIl)#wNQ$LHhG!AshGc0UzxDWJU%b_m z^%n`Z&J(KHH1)q>v*W>p)K$DJzI+O!o=)*7w}u7{q#lL_H%r|SXeKkQlE3Y6 Bo z&;_Pcv38HVtjeD7m-9jb8wlBmFbqc#Va~T^xS7xKeNRpYmv+45)U)~Qm2z|nGT|jA z4K~~h@;FzNJ)K(OgqM09pR2C0H+iEjc3d4R@nIm&ugdYHowuzY{PG{2E&CGZ(F+?8 zKF;NR@>NB1+|h?;a7gBa+Z>O$D@-eFQ1@@dOU+s#IF|U*(kSE15ub4yR{Sm^Bim%Db`^= z0(4RTkVkN$FLPG(ZrvE^JI7*>B*SA;9?>dB)KJCS*?2G1t z)ZrYNutx=n6Dp&xAX%Kt375W}tusFmR{u3m_Tmj=E38s_h{?Uz7n!m!b13&0eb?-$ z*MkNoITHN^_$!~)YJAZa6K62jCin9-mQUgcYqk&=T1B^ALQ zCAE=_2zIn+%y{`ArlCL8Ba-Td3PkHXshWWwd&_^P#GX_$jMBV2T z$D9*%0a$j8@6i|Rkcy96IByL>P@z5+`TSrt>B9-E(UI)2WVR3Klx!j4IBtMhO_$Zy zC({|6J;}L!^XpBZ-4oi>vlCwoXsC(Y2Wa_HL>yUx21A9wc|r~)kFI~LC{|*1As7~d za{6z3kO*v-q8{}?v!DdDrlnO<)whx?;CzLZZZqSJt1aenin5)!*aB}pr0dm=EGZv8 zG!uNR5uH^;i1ogak4(aUPo-{w;G)?YhAAT(?J7(CiDjGv;y`XXM&<{5@lUfk-IU_Y z+CiCJsaYQ2U^Zh=R7rFGOgQ~Ij(Gc~_0c%(*e%8iBv*dV&KaGi1=v(I(+lTHd|sC5 z&}<&^_Y&qOLm!NS+x|csmYmV#L!uebhaRwRYoL*pFsYW%7m-hRhzNyS^zgJpEw191 zm9L8A!K{UN$3sm{^+=SKzW0DPvJs~JFtC<Qi* zxl^AFy;8SGsszOWM->Fxao7e9yphHfKgmur84#=|bFj8^e`dMY3WzQMasgm8z!i3*rnZH7 z;0fQ`PbqKg&61}3lQ{;CNAA3W`pHX@3c0xgb`Sy2{#eZU2=_b zM}xO%Me41>y2_DP{QBWNJVlI1jmmUC_J&yYXZJrDIDTR+(g{2Hsd!93n=E2j`WW3@; zPMXc&wd;h%m&;`ST)Wg##CQQW)-pUdCAtL|sN2O!3kNAbXMZgFvOXh`L z7oj0MF-G|7Sr-2lv-OU^x85#0)gbfED|y&y@1bI_W2cpfo@k^L?_R$-ZMT_TBO8$e zoT(`h*HRG_Lrq%wTP|ygrfVW47u6LLw|2HW7kLnc2!9qD3Bj07Qxw}j4A34{4of&UF+TsioQ zxY&6sx;-Zan7!_W7-z}^H{2icdgfmEYhTp*qlF8n?}He#%Yx_=TKa-ZUKt+P=aO-4 zKptU`R#ONl(jQBa8T~5;ALsXsM$BiO9sr>>4`1{K&@c^-j@h+GJXbIYkUJIl^T$Nd z&Q-fxMcF%X<~_@UG(Pd4DRtX-1movy;A8LjWdupMcJ}R!Xw!p$biCvm&s*JdnQPlQeh=_Q4-}Jcg7Q{ zd-LIE{QaBtT{=Q|EOq*GB=GLhvm)O#rH*xFb+5zb1ij9rHXOJZ@@uKG>&e_H;;oVl zHw1JI=QH(mPNY~DTYzIhIwQH;r6h&=I2c)%13o{F_fSY#y~OwE$6==MU&s^$$v-Zu zb=_0mfpShVUr|~Tv_tHKLV>W9bnFjEDa67H3*T-0jO~Y_9I2HHR)p^z zb^=GW0CczuOLlpYAXrVDpgN!sWlTx(c3%7mIt+9GQhdL^q0Pu<$`P?o0tjVP#1HwdW=i80@LuhA?Edc+67I!XPY|CF! zS6K`ycP%c!sQQFk4!6xd7s^9Tc8nJl z|0dh)=8-l#L&y@q&V<&7=nBS@>VHOwFFVrZjO{GH-low*UaBvU5ldhJ`D43`24H87 zlof%_2gaKtmDBOdaoaQSlkaJM43C3fD4(U|C|D+0+;xAj|2oZT^S_gn8x zLO&JPwBQ<2QF=y&)_vdEZBg4*_;wSFkaGP)7>UUW5Roe$M1aELmKSEZ>9RUeBlf1z zIBSjyCL8}EpFi#@p(lcl1r`=Nk_jcw;TjQjT7%1e*bYb(yY6mP{hhGPf&Z3)bhpT% z5T#_evq8j5N1zW9f>hi$Mn`nUGang?pQdP0p_#wBTbUR1f{J8li@ETmv`4-uN7}M9c?NYiv2NOL_$aNifL$M7G)Q^C3}DP zKcszkJk{?Tw=FA`?3qxMJ+diOLS`JBtT^_TO-3?`kX`mTwqtKacGkhMlfAPY=XgGc z*7x^%{&`-{e_qaK-S>T6*ZscU*SOoiSTbK#3c&hBRh@4%fJaROt578K5626$f^UT`VQvUuN@zqw21b#xqx&QPJ4ABWpxzM?A>m( zoBW?E0M|e%J9cEkH}kEF|8QIRzE5qk*u{=~UXus7mAxQPQr|hD^LzaOWMdOvSnBLs zzp=CZsb7*@a=)2>juEx>`kTv<>Cqx7(Hy>%Bt5FZ;-hono4Fl}u(<^Fz`5ZS{sbbV z1>fTs{IEbY-2&JQASwXRMrfi{ji|lSYbS5}i?77h%V*2Z?iE#3PviW+_S=fWP6yV) zP7jOm+nSrD7f~pb{mI^n%I)iXX`(J1yPYw&oV>q(Cit792Jl+|iG0VRS8sf$N;$Zn zUi@m~;XLXtndb9S9#BB(^U@FgADDWY*940_NdD%*p>y+yAQ8}+gU*h&4Y4Lh)J*Bn zoUAcVceHk}A~i4OxJG@{ku~tOzcUhn%X@Ewb{b$l^l4}6);hQQXCz`J>J-QMdxwRo=llM0ycfn|Ez82way>4U zKvP(s7(39SZNmyQNIg7WERZ)h%j2B#@Nc z5osqSO3&b9&6I@kL)MaUL-%_rfUx)hIO(}vG3!wO@8g>(0lu!%Ab@6NBu>@q4P|8%S31FQ>D!`|E8=0L@qj3^ZS-dUcBcw{hN|X^|x9H4KQ(@ z1D^|&W$^#G2T+7=2|yLTS>U~yTTkG3C#(k1onu6_UGd^Ad8ms8UZU&U3)$|uw|#wk zoImilQuqTb=?euw{m$hTeL2>}2mJS|7g8^>2hVK)+J8GA{NDh^?+HVX_P3qb&H}h( zz_B?84KW{aUyY-8#Kw!!oL>h2T=40~0Uh_Eb}0o)=B}@+V4(lA)}0Hgp3ExrWv#|Mk@s7o`*aepDI21x@ADD(35pK^zG~-< zjuZ#v&m$kRAimnJ_YNgLvK>UX+TFu9uO<)efM%=&bACiFTd_7zB{{9PT@5c#RS;hs zB&a79o%e869;GTaQGt%r4NJUcVn0U_qgB%Z|$ND^RRp*>$<>=!XsP@8*L z3p|)=y(qV&x68P4rHf!s*GbyKcD)EjUanl=Sj)8;ws!c#v_O?{w4mrW|7=*?o?`P#ywB99XDrCK3|oNhA$t%aQ9Ph2PB?b zq-F?bCsHMe&Pm^~=cOCzMra4XO#Kqg^Qqnre+8xI9h%Wybm-Cy?mKcwL*D5?n=ecS=@ic z;hU-%Wkzu2;LGT~$db?u{b6;%T3NMn)SZkv#Y080Dw)mwA{KtFaI>+}mnN)Fz*`DMxUO$jt z1K03s1Z~UvTyP-~+R=?v&ZCbWuQ*d@?-$XUL|``v5==(mz^wHAW+5FxIgLh%f_u7jSZQm9;OyR||mYW916Ia+jXm^Z)m%e~H|Y)SD^c}~m7 zB{y%D$%7m_A~(C4D`Co#A2`)viU`XyvFWo&d&JoZ61X_n+DQwH; zV9K8WTz?S_QFF?_%Ww~*ZeED`{)SJw+-$e_D2upf?<9Eb%;|*(a)BBGMRk$S!NEmc zaYT)K`c<MA+KvJZy%(AJT^nASAD8^r0%G?HGje@V8MFit8wK-RflwVoJd9nV4g2V;PcYMO-7=|Q zTFG{2p^mbMDb^69{ZX5|f72gIF4&)v=hSh@xNv0ZmWzCQadr={CeWJ3dOab{bnzo? zUA$Y5ocz6KQuiZ>$NN8EwGR}58UXX;BbGPNt~e}I2icfRy; ztcEF7e?Zph`W}fOc&TYzMUZTn6iB3RrH*eFC>HAQw-+4C}98FDH z;18w1w@~7mpSNIGcS?v2>a6va7qTm&UAj*9Q9~@U=t>>25UY#SdgV^JN}m$bK$$a9 zwDzUCXlrE~EGeJ5;7o+Ujf6QzhrXVK7PDJONL-^&2>s^@K&u2xIf*vm>$JF#g()$F zo&+TuzuN&U`RqgBey1SNzBI>%ydCSr!Os(5pD(sW8&RO8BT?&RWcQ4zK7-(556}RT zSpw1tzl`4T0mi#Kt@mgAstR!?_zs4l$y2-+26_f*rjK-P?hP+Ve|>Oov49s}Lr32n zQrdHT;`F5BZB~ASStZMoUc4y<@)-8zP>H;6Xmvc!*`cSvPNDC3oL7gVLR&%nVPS!t zx3h`kWN`A!UI#Lt;do0KvG{fzp@IfCD}B#V5+{;;iR_J`*~+B9)_DjM_TEMPA<(1J z=GwarMnw6yz5IC6hALiLpYsHFV>YVJq5R~XXK0_WnG%#?FiZ`)gAJxhVhjCQY4M3& zH+!tBy=a_DFv5xKGeO_x%Bi$w?m-1#!GB|t zzGnC~(@vk%@zz{Bxt&jBl-K<9ZspaAViPg0t^=# z4yJ9mhqWd$XhJOFM#!F`In$%(=lBy1u;%ly!3S5Camk#;B{~W+*v{s<)$d2_Ou}1; z{(B)dIEMW`e2^gc(-jHw5sLmoqIzkxfS*G?o6M5R?XI|Wkr!nKGbfv|pAJDP|_57vtHu3CZ9(j-8LN-6IN57JRC3!5QiN`LH>W)#-Z82uK5+jhS5Wi*tu zvJ#4&%mbDESUh^E-`TA&O}SC7m@?4)WICzb2S|Ij=C$%_V&|GzFzU67w{qz8WER-C z?ct->$hKMa)uv8_9eiK>sn)5lao=%OT)miWM$t#8LtVqpWMti10D!h<-}noz5<{LL zRn~QF8+kpa$Gz*SUeOkv^^ap07vS=Ea_I_AtWvqSNX26&eO#K2m!j7(@bx&#i0#e~ zVEH0`B4yXWQ0Bd-Th~Dk<4C~<76F==xOwST1XqDZ=%#~_S&8Y^i1*JAo=xIKTVY!h z(TX~5uWmOfyCrPK-*Jo_&~n;LumgWbq{tuZ7*a;*Ixpyo5Jm59DfF4S;C!3kqU!Wm4 zrEL@PfG9u^vwKouiMS=kX!T@`!fZB!D}2R@BokV5U zQ)1I7&rj&>lWC~7uFCG~G~-cDUzt9K3k2BI)nWAIb6N4`A6a?SwDZ$Y{n)JcKlb&P zd2=k)f+vEzJeJZ<4|mHviKe$}tv&s-E1^YcEBidn!@T?OzWWPkpS|ERtTx6r@YCWr z!To@V#`YF5Zr@U6WwW&kk<;Me7j`GrgTVL%Vj$T>yUoWTPYq zig)va7b$^SO5?-LDavGRvd~=dZYy?$MSdk63;P(?!0T_RlRLY<9q(f?v_xYlCvck8 zT2Cq2A?&p8DSC2mCjlFW#s~>tkYGzzA=$R2SMrDwQ@-^lGZGgP>v~-gPzEgv&_@ps zw#c&W@~H&2I9n{X^YorA=Vu70cnxd(gLCQL^m&IUlaF|S{RXP|v$M19fr1y+DD46q zFrGEmoj@kiyB;Pm9BnLaPRjRLYq&4me11r1YbqpE_HUlrS9+nyo8Fsk;5;rf8>&D) zHtsdh_15U$gB}-djFP0>L@P!o@Yo|CV=v#k{!rE(@1aY^Hbvc#S+a=hgx@v z+)Cp8nY-3CwuJFb<`YoJW#hPAeWQ_bxDaR$Mp5hFNzdyPie96V-~h0;r(R~pQJ7Kr0ap^_asT7ydWw#OZ_BOVQrlwE^pk3K1n zFdPC$7^+vv>*fHdx*SX7))p(9ETmC2unVuB^@P;&lC$_Tl4L)+^Tq^2A%-wIrK648KeXfgvDq z?QxbVYfTvGvTnf4^*~GD0Bj&u}lV_j38ih1Gi~9iddkE7>-&#^pN}0ldAlFSbSs zRs1r5lv}eEh;CM)mPY`$ulW5mHtK4SHb(%-e@W2X)`V@MVqN~ z`?<@{a`ltl(X;%zG$T0`-Ay_)trekXnhcN;JeZ>U0Ds3NW^Pk|o=*tSD4@1;Ci0wE zu+d@Pco_QY!tVwcF!Q{T-T`I5b3zy05|(&a1x*bkW&P$#fCBN0^w|qWh6-Eo+{gdT zP6zXXaUQ4^-BZBABLF3u9>#OEUN4u+OXmFI706;11@vwpmkH5b@^|<+Qv0qes#dzn}_QtpEwSs@Ss!s~-^X?5Il8w>9f%8R7!7R*zU5Jl0%@G)_6K&X!U)w%Uv51y(!5)$ z%}Y$7Y&KrDLTLW`GuWha3?_-1;d|>a_tlUa#Nw_+jTZQAv&}W7ru+ow-{B=wMSCDY>QkRH zy2lc`y}W*9z7eFUsFs>>8h##6^Gi(q*Of6X{yS;ql(4P`lbvL9dwDsn%_}*rMrT7y z(ze?gfYg+QJhV{B;{ZrMggte>EuM zpGM0qTGuoZRaRxz?+}jpUk|8tU!ygIWNM zN8625LKHe-eY<{1f?LmdK+bq1|Ct~DxwSZl>A^c1_I7r5xj_HEc!QUgV~yTovpZ*1 zheFLR`U^EpiI?()@IFUi##YmlxaZyy)YzEW6Qy^06z;U9NaA9P`{9l5Fs#Ah&BTZ2 z%~`==-)Vt!axM$b+NY%BhXS zlgzJt?_l(u)OQ^>`_Hf}qRM4tY?t>|sJDPT?A{|V6+=r67c*n;2OTO}u$XeE z0rdyS|GS`oDfrV05@5p57xrHY`*(T5ACr4-J^p6O1Mb4F$i+GTf6k7hX;Ji77rNVg ztqIR_&0k#5OK_4a-F2C1Jbk{?A}7iCx9$P`U{}mc0wg@yZLz+~jx5vV$7HgTW$L_@ zQ_-DSIDzzY-9AAwVaf0tBtoMJ$oU{@{iN=iysroQjowpoFSEZSD@#`mR23c`;tHc9 zoxeD0z*%F{iF3+H8|#QcWQkZfO(!mmu{r4_CUG&nxUQB&-Ijk{x-$_O>vSDP;XxIZ zroBd$9yC8d=-XfUD|YbvQO8*Xo>d|7qGuA30)ju1JNf;KyJ~=yOIsgJvUxU77oRSs zKq$M{js+%RSG|W1iLE$7)}1;T(n+)tWtC3fyI|ihW@S!<09=Rs?2&<=;lXZW<&{J- z|0>XiA+@4GVyOWy@iM|h)N8LuK$I=HeB9U2PTJbn|B_15$~`iDt@0Tdbqn0fmqeez zAj#Wwph%Ty7QQLf@$FOYK#TaO0F5~3{!-70lR)BH!Om#KRQRW%oFY6<-X+OpM3-A* zmXD#^`-AE+>-dACUE96U^6un2;n41!{$NkY+LUSP{kk)TRFCQ{>Y4TjdId9-se-^# zLkW&NmFXNv88_)rJ~J8p)rZ9svp!o>V7(y|DNNW0bTbv82tBb)Cq9jCODNCZGh-Q! zRqicbqRA*%wu+P$SL~`vaIPouI3XlE<05@z+}h-H9W=3d77TSFLLN$FD`nw_Q^ep^ z&q#MaCo?9#0{day)~fTEy7$C$Wb@FD6AQ+n^vWqjo?GIITt+gZ#T(32M^e2F^Hcnv znNYZ(XCJoKjUWfp-f|)nlIBUPP0QE#VoUcYB_~`YS)t>T9ama(O;WcEcD;bp1KhdV z;Jy`&vj(VLv(1wz@BG{Yw@pM8&ezt%duNNfQ;fc|GB+E~T=&nM(@r{cCvQq^(%f5K zrZYsfBhf}1P$qvU(OnBR*uHtRUfSU1Hjc4PRFpyNvAkzW)Hjg(m>40XaBoH{BKip> zT+8#@*jAOD(Aron354_B_KOV9IfM7KM%v4V#Zm3ws4y*p&Zf0-Ae@neWj*};?X7T0 zrygUc2toFFySw&{wan^QlhQp^%s0C)$CT0dS^IR>r1L);@XqIci$`5#R-^sM(wH37 z<^=TAwL?PgoY}?Kb=_;s%RNvidZ~-`d=4Q&Dv=;N+UkaM8YwxOBM6M_+d08)`V}9i z0@Xb((M*kIuo$_T1&P3iS(|-TBYlB5m2KcpY+ee4TAek#BnE0v12u!9uv|`dCj`Ms z>P&(dlQ&3XZ)9=Lzzc-Pj;+Gd?TEr22kI^QAE;C;_N5j`;%4TPc#TljnA^-#85atf z=@!)Vaq-8gkwOrsF3ynvkGrUoggn4zI{}Dn5h8HdD}rz?-`=&!_|00k&yrbus399e zH#@_f%ZFbiJY%-34r%Qgtc0)SG)>qw@XLD)`yicM_P2me;F7z?b6=-UqE2nD+2|f4 zwj{W9C{E(-j@Ao1h3E^MPQRPQljNxT__$^09u;V@A<=g2!on#@LLw?T`8!K0dm)87 zOQR3;Z-apTlr0ZmwPS;AA;6^==FNH2U3$j0qrJPg`|NhB64Q5ct6$`dg^cvm*LZ!?T`CZ&CgGj7o!y zxGrJMc`38(vT)RTF^X+P2I0~ZBI(n{b7bl3C*hx>#zh`Mq{4PYmN|8FJQ3X=wzmRG zk+~|hpVzpBnx2<$k_8Owg)vFe3)(9;Q#CE*&+nmPGA%PI6bvYaQeJ^|DRyP>%gNk@ z6bcwo3|^}GuA;&?w*;A=^rO$>ENVUf0>tj_mH)j_7+79FLoBU-{_yVTbw&yqHh z(k)W(z+&p{YIDUS(+F0R$AQ+|B4>g`&sgV+@d$ya11XpTC;f#M=nX}4Z);*D<&!|- z67whm9*RL~o5UY$J}Y5!ml7q323s4GY-`$(Q46DQFOrsxjW(FA3jNC^*vxFypDZ-f z$6Nv<6uV|G$2@eqVO028*6nhR>=haRcQWRR5!s-xgO_~Z$cEL(_L51ADIaxc>8DGe za$KRxq|#T{M+S^~DX2td)upCxTgKBW?YuQtm*-n6&{+(G$}Pf4k30t!wvr?kKl)aF zvXR+aZ*9oqX=;PaRQCH$0}@vQc#o)0kK>pPj3S8gCA{{rz_F^l{b>I-^7F#M{Dz<^i+PjC(&d>&>D0ytE2u0y zrU^ydR*(Z?^?fPSgPX_1O5kb^1vcjmeY;}n-NpC3kkF47X|IJB>ZFWGW-j|~Lbi7O zYJzFhMfhJRcS%?ZvWErrSr|7622?JvFXg_DEYCtxpUMm}yI5x-(X5)AUc$wgEv*&|@17-4?nFi3iei zPdGGA#6ST6P}-LB7c9IvBq~_9Ax3)k0!MlwP_XdM9XuFnL6?+erpE$gR+q zDCjYkbmL(Z>=(0%2g`ovdUE>_`H&5O{=DXlmDod{JPIGp=sfctp9q;rio*lz%)2h= z`Z!Tl6SgGIDy|I<=fH3Gs;zy;S|{i;*y++@MQHGB8UDd*6zbIXg&7eV7 zassFj8>~+D49qQZXv>erBiuZY6^R$e1_6an!ZEXPR4auayK6&@xMV-$*kBcuNJ&22 zztFj8rALhUUf5MmtZHeF)tLR9tXN8%PZK{VKFy zfG)O-Iq!6R(e3N6Lkj%FukOgLOfosg7AUkpvq$ z90iP#9}@;gW;XS;ji)D*F8OEV1w-R(7F8ij+u-=@RsQh}E<6Ven{~Sc5QZq(ZH(|FHYNP}lTHzEgVxfd1HD@-X-)KV& zRyeJ8ZkX~B-)+YL?;=Z67>0+Z8$n)qQid{p5k9^VHGMp(S&|&1iyburS-p5MG~=sP zYqhSj0|SEQO`GH9zT5{#VB!wJKuutzIv|AMSse`7u3JnKDdfF0$H@B8k$1+7qAjsm z6@$7;ownigMRO-K(6#wFGJ9(!#ph=ukJ;2SzrZP<=Zx_9W2kK(wl>IcLHu*9yV0eR zkr#f&U4ak$LfTMN@%XUHlP2As>RO3Qf^sF59m(Y_7?ysYIq?(^sB^*m?+P%uJtfvf zX0%F6DqAwyZ4c6_9!b;Xq!#h?-`s1uEjp9_UZ_B^%PgcxDx+xe*=6jzDQs93{G^bPlikXM+o~WU*XCyU18mma3@4#S z^Q7CAcLSg{=HWN4QTn;~yepwdx9slyAga*UU|J6kcgmYFNWArhSJF@?DYj7`dTje9 zY7FtB_?XG^EwH9OC2T5nr-H?_!H}^huj_wuHmyiD)$#gMX34KQJ(fQm)IYo&7aGCeOH zujz-L05; z%I#xzRH&qL?)E1_*<#|T+o#O>cxfQdvYNH4H=kXFK*>wZ3#mYNb`D<@`!dj9g#>w4 zycGnDTq>%B?W>yUlUddm>J{soTN0A>hVKLr`mpubH;N@yaFbQamWo zme;~w@G)jAv*M?*$DvQzy;Y^NFKOJCzGkFU!t#x9mST_{_ZgOyTt~pj5=mNvpoow` z6h+ScE*j-y-#RbZ>)g!-eu8p?@A;^6BvysJDR>C!hF}na2ZC&34xXV&5!gK4RK0e!1q*(Hav_tr9a=m<^Zc1 z^=99XOKPPRS}1PIydC_Uks{eOQSc?}q7DUmg@*Thn`IXhdn6KlF^xF)nqIya_$tVz z1OJi63+zsFmU!R*z7`cdXIa(klBbH>#0;45gh$4 z>!;R{#@8#c@Mn%ijwX5>XC;;vh8PV)drckK>bYJoh{7w5N=+a6LzVE_7qglZ8?uFb z`@%(XB5+(!kSSM%Ir(rOUOqw$F5Gtz0O=+~V^J5I=o(_c785*6`lSM&8`;FO50|y+ zsZ%*c)Yy3v+uKEMf^#s1fs@AA0|ezD&u^=;D3xUOv#L^iXHfwM?FnpX`EGEw@sJNP z;fhgUVSlx>NoHcN?^jxR8&DtsBGbtG7RqeO;drerkuc!4CEse2e6L>fckvO*g9ISmf-hlW-bgzAnhVM0D63Fnt>B6w6)bJt5?)+})E{ zDYQ#hN%Jw;ju1+RRR{nPXi4^^7&Tg4p#UD_h+7t!RnteMl1y(b!Hx1~OysWzrTU-9 zSFp!EQPvn%7~qC#)n=-3;L2c$ZB%(@x9A8r9pG1&mr(dAf)|{9yeYhb^CM}JBJ}|; zNa>^6s4-FaZAYMSpr~MkTw0sU?|u^ET14SYXcQCyNV$OqiRyNGQ{~4{Ci!O8F;xvk z9x|Em$k*NLL}S;DhaBK`3Wo*CEwKmeRyimvqZPVJzN{xZ1!JKRk=i`O9l6~1wZt#$ zzmZmy({N5Xu_lUsp`|Gz}n6lv66H7wHLut&jrGpB=Z z^RZUbnlSIIXdV94&h&jZju@8)Y?;@H@|>yJZ`u|S_~kN+KpWaWM@a*bkCA{68A}22FmZ%&ykcn z{`bZ0H!--_8{e(Cd1H@fCVT%JT%!Bza!AS=BwE3sXc-b>xUL|*S}OlKE`>9rI8+<4 z<1;!g@gaGu0{^jG--&p#l5-i+FZ%;XDB9hElnxT%KJg9^zOR1y$Mvbf6P^YE9hEGr zH(_1`Wl#6arb_9?7RuAztO`t=tt0Vc`sF_>ZVB&lr*bW3{VJRXBpof*4xiq4VK|H2 z5v3H9uUQvEh*S*me!)2JV);i{#Jg`TH`6rzhlB^C1vB+ek}(gNvr)v%zdXLbAbPd? z|DIm{ivzvLwg*-OnCPA4KXCxw67;%X1n(VHM9So}}QHlPb=AwQ4!opb%@@6r8RItSc<{-UHGyQvH` z_7+bqb_BLnT(6+N4E=p*rw*=NH-z1n@)AcGJ55C5ydLFG0OZ1SpkoOCYnd*F4iHB4 zM^!IJo%<>Koffk~FgVe`04?x<Si>QRM=?cd=KYOBKumE}xGo%pZCHv%94-GPTn%XTx~5 ztmC$}B>B!u@)VR&x41gmX<@hJdJ@NBQ7s{U!!g%!*<%N=@Kxn?vcQmVtwS4D|%U!J9_F_R$FCn-23EE z$+2>khA-cBR67%H68v5SN@AF8?Ob5d#_PHedS-pB0x{dEXfGIz(EZqtiU>-*9UZs% z8Kl`DYO!f%dC|}y)h`8Bw}9gRjKhl$klR!mG1aF{`O1fO|t-b{+1faVrg1BP@EQJRSU|*lXScd8r2rLhyOL5yRWk@-0Lg8wMPr z1=YbOEfpi?F(s6SVa-U+-!vH%Z7Q6w*4?TSq@e0&m{EqF8QgpLHT2o$eg1GcmACsd zRm#|)g7D4bfZPqd@?rwCSe*-$jd`)QI)|zDcy~vf0@D$$~UvrD!!Dwl-5y?RU$M~ z#m>?hgytny9$`tlI8s-~d z@%nQ6T>@TH6HM`-{Hh<$HprZ5;jP<;+lK~1&&Ca5#Q}eZaUzUYt_laxASZ|pxq|vw-2EXL=`FepKa4F14k%c%xzI~q8bBK zQfaNHwQ_N5I#G5ItG;u_s#-*QRFfEz$3!p!G~p9R7;|RI!E$6(TQkL#Xo;IOj{576 zuNqsFxA{??+SgSHS6&k_gLKUr?@=#LShSaY@l$h2T~-Q_Uz>c+tjDVQ()^8-9Y#xv z>M)ONi4KV~*WT#+{n5s5jHDnr{drCP11cC?QNYpo`m;%z7C7I~lB<(mj_*@)yl7X1 z(D)@7mNdiO#AU=*!5crQ(&Cs7X=NXz1{}6|=rbhgm)USDVn71(_lSUQL~u`-#t5(9 z`{uh46C%|1zV8POm(#$Jrlg8d8X zf*!+PI-y*f?2^tJdy|X5WcqIMcLe8ZNP>u#rkCjbJ%4`)JKk<3K)clYL(Lz^8^qX=CdN>e^6JD|A&XOKG< z?RDp48ytzo}(fuyHL9dPiej=OHI5#7! zRb{ob)Tcxfb%B-X*(w_)k)_o4y8_3{DxrxR+uNlR6}l@!+m{hdV8ZZ%$w$oxhyz6W zW4*kPZxl1;&>BXw?6d|SMD%-9l{?6C$1>pHBp%VIbL!!CRDDuLspe6xK>=gG=uAk^GP$Ja=<3$%sj7}}Ud=q;xH+GZlS;%X)pv^zt`Q2} z2#Pc7N^)T!G42A`oMZ42CLl$``+C+VoZmRaMx%gve$>skeM5C{3HaxvAKjK zyw?|>z*C}$17Uy3dRuk%)09tluP;9__=$Wy8o4l8?Znz$fPw>EsY8Opov0#uOdqUI zXuvar>t|IwkE%F;U*#TY+}T16a(cCK*U8!CE2g{iLXZQAZfuC}l=ej5ryY7H9s+is zHokGcx6`4!D4*utgFL6kL;a?hb@e1EU0r@G*sx~DizenbqWEYv`im-um`T*~q5{ZU zVsY~^#KP(cruVepvCCCfWEU*76tk?v1PDOMKNB=Rurm9y#-;41Y=aBJD#37A-EFLE zn!PLq`{1VsjK@ubwosYKo(7PzGJtogb78cz7xCuUXo~XJ+Zn%;4$W~q26IC3kOBI>n=C9@=0k)_Tz*;a7WIi zgoCvY!YM=9_wys*a0HY4kO2@_`0ig)c(#Nkj-D}+?1}2#p3_W?-phx;3}$? z1qo*H6UN~@6n*%v3_~xVz`iHoj)54l6lVLwK*sUH`#J~#(DoOORscc>?D-USlvM0|6n^rSuPomQ zX3-M5zdy)@8h(mMcYY#&eGdeliJqzE8^1>b8NZZ*3O2d>@r$acc1d2SE+~o;gD+SR zs!I4m{9%?S`X$(~_M1x*3HgWKu%|-^_+p1vuQHHMc7bbly?=atN6?}puVExxlD=<_ zm1HM4nVg9i>|z;kj|hyJiUXoJskp4f9{hxp*gL@8YVl%p3$Y1^Yx1qoHjl;R$7VfcH~kXH?Tb65(H>TY)Fvn!oZTd_3kMu zOnhlz32$bpJ-T%(`_3+sn=b3D!X^tP$r*W4F;9xrl09sgin4R*l8`x{(SEnSHaKg% z-uLO_KDShSn=je-gVX7raBFBh!N>@W#ZaQyz7$$v=!+|3oT8IUn74NyVIY9{Hr=Ug zkR%q5n%T%IM-r;MHotd|5fTKo3JcH{CN5zsQDdZu@5?9+dB9c{_Td#_bikv1n}Yis zLFygL#f0x(r1hZ>!LxZjkY(M6P?CMjQ_^imDqY?!9gp#`bA2etIxPLUjd_znegg+< zI z1?oDuYl17jM8&h?t(5Ey3a7KISd>q&390ApJG}EVl+f|(z~LC8SAbyG#WKHx`QG$J zE%~CvU(j|Z3Sz#;P<(5u&VbEhnU?-E%qVZNK_5RG>9BH!p%#`Hv~S^%wyKO=EoXMQ z@iM2;xv^QWX33e&z!}-lImf^c>{)<~U(Mq}s$v>RBrLeDc%Ay+MT-gb>5o zWIuAUIw~dfk(k}7uk9+$R>R@6136U-C7^(*Z%Qh^TCasy28AlW!4Jm3G4nYSC4ENg zb&__l#f#k7o~V>t(z8~>DjyJu2hubNv2QG}r3^mI0^E&8L9ol!?O3X#e0D5Yn07hE zft}EV4zel!i$lK zVnBX-b;14f28%KJrY8R5u*E6c#;D3Th$QQmBAD#=!M+4MwV=KjJHzFS=K&>}@s1L- zw+LW~#C}1DBwmbXW__ii0p%~rR){)Wb21%ZS=XmPWL&p-3>u|0qMJVu(-rMDc#VUd%Skn0RMo! zD9IEA4&Ss%!OU-y92_%JW3ri1V($Q|wE%rn?D7}%p(K;CBen$i4q#JU!2t_#a$nEB zy=_lac29lMuH%gfAOPk@2 z+khleT|#%R?ey+H=kZ*5jcl^CWnjjBtkTWq<>rv7%*AK!B`<^~L`i0Q?%4M!sSdnb zzkRiyL=~~^A%|nfwc4MB@jM5gLDxBRD3gkTN43MYd`8n5$n_bSOp>s4a>Z_&T13$c zsc1gqDwF z=RQUW;Es*26t8>?VMP>`EiBeno$!t(IrZ<(kWx?~yqdQB?5Y}mOxA5$i&}bj*5toE zn6^Fm2K(MYxryZP4fj@hkV{CtdB&U;Fs|lURC^1}6<9B4Iqk52s&VJ1ja zoea9B=pyfWuO-Da{)chuw*iQBrEHFej(Bp!HomXypP^R&q;MO zp5{lLZ1yMPt6+Y`0g(kkF-&`B8pzQ^#zl@TTB$VeLI~0;p7z+P6Pj4AWOdd;U$^RL zd8!W@umeW}{smiriZB}&*3EkoK+u48&c?U0l1;lKyWefHmKF1^u02aIl+^Gn29-T8gF-4thHO%xltf(2zoLEDga&boAoa zzs0!$32pgH$e+tB38+qZ)7GJANq*@h)s<^#v2TyD^H|Q!e70Il!hJBM^Prkv#(Q&5 zg$>^?}WT|2sFE#lUmBI4AZ(uX;!a=980 zAD@hrn7A5+@`>*>y>__2&$_v>q2!{hO#1Hv3L0_xzF)`h+ea;1J1eU4PTjnreYw4C zyqt&a649*S_*>k6hfM<7y;{znkoKc_T{b#>xi8s4Mn;Cbl-RbSAXm4ap)3?YDV6b=! z&`alJ_XU^-!l4)*#j$mKo*Z=uZRkW#iW4)F?4W7H45a!o2GQ)g6ICLb5 zpWPzaovjU?FtpjDvboi~E3f@>6xl}o9)46I#=os*dzw%_P`*C-CiS27a*u0G?CJw= zz?!9fFT0+^?CU*!mxnWJplGlSdEbf?nO%K5bj?!;-{Mh!%eab=NBp>6pgA@xjw%? zbqoOE+^-Glw#cF+eE;4`c=@J7P6%G~zA>tBpwqKU1E>^mVe@QFp%#zp&?h)zwZSH? z<(G%HfBMvNl{+=Hh&$y^^v7HcV4LecT0IZwfgn4Uv>s9 zBFFmM|AZW3OxSkR=WjlWtk$akPF_?|F&rBkTXU~*C!(mlJZ1^7*1yGu3Ak`$w0=>L z>#+FYVOe}bkM~g<NM z{u}tX@|ne__9($3)gd0p!#N=U@fIM9gczjXoGibiEpBew>o$AoQB@0 zUl!Rybx=-ioK0^OA$#7SFRdm;+Zi-OPSPzoXiK|XZDIJe78!LIvF%yaN$#LfQHH_lK7YG*g8K`~-kFzfS--$L_;RN?l?7OGl)#4%|3G;t2%GWsUu{v z(k_)MxYpC>TfCR1=Y4&Q9Iui%3hnX|wky_Bi7sy5O>WhO5p!Zu`y_2}>4P5onR`O1 zj~hewZPnkw{`*d~Q3yd#D~fh=^$C;Gx9sOE6zT7Ea{J3>72k2mO|l?KyibpdGTJ&K zDPhiBM22N2s;@Ykb?T>kA1;8({{*YU{h``b=1NyN<+|^($Oj0o;ivWxZ*`B3x*IFK z<4d_|6Olyt_HS1z^!llnP%wi#JfGIp7Q2u<>dLGLpvSt^nN6RsA6mhFO^5-6__pzX zO8d^JrrK^>nn+Pmq$7x^2q?X%R7DW!(t8s`ARtJWUPS2~>7euynvl?hph%S>Ep!k; zT1X-_Kp;6gi0|in?m74Vxcp$S89O6;KV?00%{A9roe5S8UFo@w^8cEaA`7)vdN`Qp zL&0aZ_dU6}j+TAt;2nG7&q;a>%b&6rYvAc6D~bl7-Ncj+C>4w+O{M!Vh6zrx(hh>x z^7CI}J!r;1F61@<1e*UC4sy87C2)b#_X06nUK!d+XzkfEA<4WVRJCNT!2tMyR%uui z*kLiEv@fHdrwF8Bs^QIJAQbTkwH)>Mz$wPy{o z_z1c%+AjFskxE{%p}U)!uD@}p)6|d7y3yJ)bz)7QScXEA1+4(s=Q|H0CFs`#qO`Gk ziWgd30$SQ}Ebu+?twTS<)mY<9m)M)VekLYgM2wPzU%TLE_VHV-GDWm_Z21nx(oA`6 z!5*zZ7=_99UP`g=hrrEX#-1rv%R#7~62+h#R+?N837mMV7vY zkdu}6?uR_k0mB2FdM+w~B4enLPr?jxTw;N`npa%By4KPCq?Xs&^@Ix@;nPi8mkr%D z&C7DJ6R04mT05Zu>FtKkm$jWxB}h-xpl7vWPtSZ+onksvIr6KUnmYSRASzoMpb>?t zJgP0!1e)NI{npKBRkSqqT-(>8&L_>w>sQTX$d932n=1ue?|{;`=*??z&N_b>=Obc0 zs!x;T__`~>739NQ^70>MuLv`6z^K!7L?!d@$Jg!`$9MfyjbzPb`U*7a010D^Oc%OcJb zORp0<=7lG{Rm8ZOlKJ-yN>65xfXY-~i7y0b3xUf8@9A0vwzSFOYrBAOPfRlT?k-k_ zwYSEAnV9GJuoUG^v`h_JiyAGb;-g`-?m6qV+iZJ~fzGnukNJwhL7mqP zW+8gEmwumT;?-2IkUa6(o5iNkdr3XPk80AQoW$dyZvj!;OWffo*AGoL1-P=dmiW{e zx+EY6#16_6BtBF-AfG18m8-u>%+tESDbhkq%CdtqQVfCAqJ_YY>V~9DPAZXK+IhhD zL099>4fc5lNUIGRwbNb{GIm5gn-t@VMZE-(fuNf|u8>X(fk#=#&wQJum40)F*0WU} z@mc7~rv7udg76K7eh=d^Il)6oB#s-}jhk%2%X zcx*FQd5wh2%j`LYM{j15TvXHcWyu(hgNVZy;3O7?Fi)%sWzx*J*Ap`C_9~ma=g@#p z3Av^c>Mf{9#W&bNq5UBtQt*TjJt@fJQp5`V+DT30j>0IVfZH@A{LpG$F4Kx$#-J4zQn0R1F-1w(7BJ?DJ=a=zgTvX#aiv#_i!o!cZ;Gsp``kbv zcq0?7%}qKE7F;m@K3^*xT+_Uit()IE<|2Y?#~r@O)oBMPnkTL>xqx|pesM(iZpK=) zrJC_iAW19-?HX!WR?kt;Yt($gf}r*36PvRf|B}A`37i!a&`KVEwJcg zlj#;JrqIalT{;K3+{ZOVU%)~GUUhm@P^-6yl53l^J^Bq6C&z2=UiZ6ov7l`oN zxHx!{fifbC$r~hp9%9v$=L{_R9wJP#ZJ&U8aE&YeF{j}_u(ek;uv(S z2<>zDsxG&crCdV9PGDVIp8i(aTD2f{PZENowM1qdvpDT|;WEq;^kd>fq(nE1gy9t`s_q;@gTrg|R;UMggK z6B!)^==uG&QK5}%GB)=~L{I3W8F}!=qR%r{f%aqi zq)^>m)BCbqyS;7_J2iMiK!y-v{+Cxlz)vUt`Xie9+gbHl=MS_gl9u z>1Ur5F9_bBxrhvD(=1(rtCE-!!^xD1$q+4YvCIVKw)Fm~A@R8Fyhjs&Pt-(kbyy6w ztyP!L9u=DtXH+O+$?PQ_Vn-f6e))c^<6l#pAqb{SaCm<~p^C0#=vk2A$G0l2J0-6x zEDdT_Ab86wG)3-ND-(L(OJU1MK7jX;l2P)32v`SQh4d zf7WA%j%Jp#ww}-l15%d1d_zkEYS4JZOi_~jCv_Qv8V+k>zO!(4nQ1f%02?pQU#Fyl z@EKfa5;f4)RZ)OD7#qgzN8{JkcK_uOXT-;gDfBqJk0G*M2nR=>g3}pl+dn;{87!dC zK}D}bId{rhxlR>S2KYeF|24LNvqI_d`9RPQb~M*?G3)~f{@EKU;ET~A0UgKy#hqTn zJx&(7r$3o4rqCu@=xhc#y9%@p*sNZxf4rkP2RWw}@Zbe)!9)8jQ-c}?s`x6K|Bx6! zhFB>}K9$!i`?%6^rij4t(x@wZLiYx}EYuMFI%ln@%$UOh8gwy{1{kTL0v;V7R%tJdamv6YzY9xj>D+ev|j>kb~rV$VM`1Yo%IoQh@F zaC5js)NH4>DIy*>-Tbw&x7E&bDeCKfl%(H5Eh`I)OGrpa_-S%-lYju&S8L&L?7;yo zW_P!+v8d?u-_OlbrMChhGsf@KPhmoUs5U&2+2t;~ip8yrJ}!=9!cng)9RySSXTW+F zfqpP4V{_i3N~XSR4gO+k%mF+!emG(Izfi4*g+ z9cc`IRfqm-S)3tsc}DYtH0`k(xxvjDGt$ z3(Iru$QPd-AODG4Q1nA>;16EVN@s*4{wfR&*!TjYr^^rz6*L(uQ*Yz?H|8-}ua2X| zg8QEM_cC=y#y4gFW`^>T+V#vMfYO;+hPi$|TAq)RglD-I`Pt`n9A#rZzL~pvd>jyr zG2qDK?~u0gD2oZcW0a+7ct89sf$<(5?D|c5Q?enUk6qpLrogT;e%<=LJX8 zu806vGK;LAehTQ$HzfN@@ddLL3F!a^V9Jd9FJrf)9&$P^HbG1sixP1dGVxZ*qFz%y2YW$tgsyuw>XN*$dt+CE;-&W^6 zGea3QVt0q^PGU6#>W35*<(d>`LL^~0*dTWf>DFs|kG$4-=C7$}gZaIil?Q{FJ6E$%(;%DQ3C=mkZ zX(B*)nm^p8ST;4`5&%}mAq3!AGXJ+5GFpsMwpa?st@I!m2R()U0x5gKI7TmjCcG_h zdh=&8EPcGO49%oj{3sDDmg4Fd?^oW_z_UE#>Z^|W_^eFt05NsiA9!U_kQ5_f1OTYw zVX2emnv&yHx0Ihd$VJK6gV4T{sZQjDjs^|<}&r=5`-*XfQj%}hy59I zPB)Px_3N)KiR$>!DZfWIXv|*l)FW+yhMCAUo>aOep@(#150~$zp4M8+ zTbRnNUJvhJ36#e?)$cc~3D75|Fkb6Ko>3L}N;-`=!Yol(u;wDtE=x=L#O^XbJ~!~R z@~*}iW%s2^^DV2B;_`#=5b|%!3YZ&+%Imo15k*Yh=gfh=T5c}F$@%AjTaBtG(Ki}F z55v`wO2qXB#9XCP++WVy-dJ}pJc3eCx#9Shd|YeQOQ#f(EN6Zt@5?mxlEEfJ%mOnY zHb3UpKb7zYH}TRuoB*_NK#fht)~65L%6*O?ZhL4;ydG(cJ1%<#+aWbrr%viNb+p%b z*y4h}i>SQoTM?1{%y-UH;X)~(csd%y6;f2x(Fa1D2Uz-quzs+t3+J5%L@L||r5$#$ zQ@yUG;>)&?x#0z_Nu_CgK-r5x zAmrG}&t%179hT%6o@<^p*x160Z!Q4rwjYm`2^)RzYRtW_F@EpEhwjq2@3L12U#R?+ z;>VAm&gXdTQ`JsnQ-AtO0<>?sNcdv6@%f|IyNZ~qTa6vwLmWbX_PGW<*xkcJoe556 z7hC7vm{%l>x*?G5lJ@BeM!$(Ym5!*vej)4m=-Z|uHQCn5FQNhwnQrNn0SZ)ci)B;| zU-LidVjpE~UEJKX-84;m_F;xDNbe~b=#4g~|IP*O0)qTjNajEcBf@d#(;KB|j?PCb`ob-YOn$1N`TTr{A5 z$1uJ3P2~BQCrz}iCz{584Ha@?TuR~Br_`R8VSKN$#3tHw^D0Tp$h^;l% zyl3HPAB4O|X3!2cG`8Xt@fc#O+)3vKw}*=I_8L_cTu8kY%jsmCN7pK9a5J*i4K|_2 zRuEeFX#~z`sb(Lo#Rm|*8o-2q2J;wO3`v5t;q2n6K6(r@iS(KhmSy;su~wr;QznjD zq5@>#2|004pY*r|RR$e3sPANx#LlI=t)(xgtEi|fFS;)r5;Q!K_MQ_!*^<-OwZ!}B_w06}G1p4MNQ#JDB3hKz9<*{*^_K;Pye7`?Y7Sta^^k4JFkE6z{y$5130-G8fC4%~OJd5^{`lElkTmEK21!f#HvAUv`r~bh=V!)&%Wh#n@9PdmAfur3O0^DEvS0Sftay^->=hya|6k8toD*;_qOH zR))cEMOBWpjxABZKWIiQ7}c0nOihQJyJ+|^4&W}PLw1u!`Ufo>?UIxmh<+1<+kJFK@1*e5fG^%JW3_T9r?NfvrB!#A3no&^`jT)- zJVHud3`>buukpAQWd%dKHr&OCU$&E4xyiT`5~Kw)U!Hd4^xn}N?eZplL@W+SyX3BS z50Na7fA3`OuNk2WFS%`Hz6e7{!9M^ctiD?)FS^AyRG#+D+1iv!seQ&+;iOxeEr?_X zM;zGw8mk?Mtri{F_^O4P=h`w(1#$#?7gLp_mhLW4p9Mt}mK#BO#Kgs8Kz-@CMkK|) zSo(>C?XlTbC`Xp~6>Z@%Wy^HQI zX*b8N+`4sNMAPon!}G`1g=^!Gy@AH-Qq^1eL9WMVwRmDkioKeK=pg*6gCa_HZ#-a} zAbqD6P;XVC&O>?;%6E*HI6>a-f%hJ_GQjJJK_aK&aCshA3PMjO0DUcC!d?zc-mwG% zU_mMHuuwVe_ZCa_~TFqpslftZ<8Duo|v(er1py8Jf zWOYL13E8Fun3cL5^-LNZy7HFyF4emsGnsPHM+##j4;Aiz_V}c4lB^l{beIcb9xtTu z<8!0vr?>qD&|ne6>v-Ld*De($bv-V=hI(TkcO1L~%;CkY%#F-{gboB~)Jcex&)+2i z@s^lrTELW>#{dS(P~?P&`5i__lHB}w>7_#UQS$X=Sl6B$QQmA$p=W4G&-+x4pV9*zU}#xmmx^o6(}Lm4CA&b!rYAX;naSU& zsyYJB6AqH_<4*#;BYC%eS!lD~%tcfCX2g*gt`4)^l;WVg2hH$ffHV9w(OtY#`jJ|p z>$BwCSJ-|Qb9C8(T7G3Wjr_v+IJfR zpG|qnvj8+81dsyr4Pfcs1?Zi+v{?uPuULoP&xc5wr-xgHon@vD*e3{T$7rFCH9*l% z-(;V_718)IewqjKP2^)p**BtHMq3kU+)1Gc%}cJ4vij-peb?7eHZB*&o7)XMlP#Ze z0JSZGc?e6ky4-K2Nj~v|_(_lw-j$2DZ~&Vt26`X%D3Ta52^2hgl52+0?r$DPBQ^ygGxGgI zBX4g6GbpZ3s@X5<(PZq_e;c@;^O?CKZ`RzknOQOGbRUuFnR0CZpG_5m8(x9o=- zmvA+QGBP^?JvaXhogD=TkRhb`93jlnvqvb;CZLmjn>223VIf$oInYQ`yh$UFi|8f; zyzt=qXasn**@+SRu?g*@Hp_gQ(Sna?sFd5Kc95v2kj~(yN8cpF#r6o@dE)9G`Aw?w zm(+2(rr*8w(rmM%XHNyz9^C>en>_@L48g$ww@8(^#Kak7xn9ndkh0SXXl?-s+qxf6 znep1i#Z}bM&=9kbdar9oB0@!=xwhB?_KO-Om?^Xah%s*M`|STrX&OLv5x1i8+L}OB z3h%Y9jV&`_zhRin8p~yG8z#8koYw#a_k~N8I@_EGxheu|=^LHHEIOAKnDouJt@KPbe?W>exmqP9|)DE?r3|+E$5H!@6h2 z=(9GfHqC!3N_6#mj<&iwBrC>mTDe=2Q>9@wviJFpn@n?+WYqjb%th1&w4YM(!ZQ-^ zsrrc@LHXSeI%N{M>fK1H^q%GF*-bLhQ}K|)_2}!f`*@biVmLh%*I*IQ(ZWc$TK^kE01s00=M7nQz0Z!iYTvnA(=8sxL#xg?{=5O32hM3F`HJ)45`i7QyqZZs=5fNzAHAZ;^xNJ3d%X~`QUG-(j<50Gu9;3|-PB<9ejrA?LyN@H7 zf1)A;tu-axsM-0VmeG_+YM`ixc)Rw(m`u=8eBMmA%>4|D$IFyNw=*mqvtvmhsbQ0P zm)Y!hcp?fIf#_!X$mg|$!7`;K;~Ho;*pPa#RoVvc0z-Zyv*25(L;f1GX0sk^WfNCW z2_&owAg>!Rcej8dTi+`hmRQEd$M2xTKqNGu;^97_By2|b>%SG~TGO>bUX@*OQ&EFPhL(ln zm3k8J#%1}av@ce`q&6I%iSJt<1;#wKl|P?10%L;Hg=>u{<(zsMQ9<8iqt>ZBpX!`B zbk0#yp_WWG;+@juWw7h3-2(^h+rt)pyR3C%cje5X=BbXZJL7Nc+1)^LgN+;voR&UpY%*uv z!kNvB8f51ifyx#1T84|2C0~gaJ|(!e>xc*50<6YwsL)_~Ryco~hyJmqph9gDiYX*K zS1IBS58uX0Nw2dK{)vSes2PxoS|pN#?2E)%Gy)}~MegXBWjYvWf9cOwJjavWI(e6G zQkJGOyhKH-KcdjohBU1JsY1hoj%;C#p;a^6v+&bxc*qs-rrB)nvkc`WA=4(^P zt>qa+rzs0$;(MNDpA3oZFilnJxQ?lD=xIm|~%IqfjMMJwtBXj`?Tf1J!8LY09JO@)Cps?*H@>7ff#tQCsA%g_GntXIeb^bO?>KZ@3z5pCXF;pLqy!mp50U)VG9IrNAXy8btxG--<>j)MHrm2KFuoU+ z+g~1^9b}k!+%!HPIw&r|phbP^1@SAo_Mvp>V}mcdl;Czu3VB z6P<#N)hG2c$Es8nLW!B&qZ}FME*|MZl%m;AXrbS5LX8R>-C3H>)bmSY#0~^sDm@9- zBzgg+xplH09F{(GKAE2h9!*eX5}?NM3IU?!5teFp!6f2X)Q4~*o>$7|WPry0SakXO zqX2Vw9Lzbn{ZCAYYXbmD{~@P_(71q z{qNu92saX-fNc&qd5#@A=K0`d{l`q}7GyA<=KC#8^JO$mQ+T8jrjmxu1HWND(9r3B zsP&^LM-n9g#BG)DH1k|!J zi>oFl5C_gpEv{W`2TDbf$xUz4K7kQy-SrihUhOP#s1h**mNC_*I~=QHnNCIxVwlzF zwX@VT-T6G=+ssyutcQTdPGive<8xiYsPXPuOO6aGCeq*2p!!3IP`bgj!^r?!a4mX+ z8Y6`|Cp-4@;Pjh>+V;E~-U)OH3f(q6Xp-N8z>|UR^SV?joDbtwexYhn%W_@GvJJlCxUqBKBcf4%B}kb(B~2SL5vC^O9Gjpv@*);X+~ zB#{1xpDF7wvH_VNYYh>NWG{G)WwUZ^Zp!XQ=W8w8hT*}8JF|)TVpRQZe6DBz`vdWv z6^=~?e8;c|Wy(Mus9Ag|uvo@6*HtRs@7sA=wcYnT;!{&9eQ!o_RET(}#KKueFPjoO`W zUx6y&#HhA)|BlUq6tf=Jj;oniwKFc4o!3)fhd*NlLlh{kygD!U|Mja+=Lj#q{z^+H Suza2f_)$_&m#>hs2>vgvDGP1@ diff --git a/g3doc/images/mlmd_overview.png b/g3doc/images/mlmd_overview.png deleted file mode 100644 index a5e5e05eda7dcff89a132cde0e8b629cee457a9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55112 zcmb@ucT`i`);>%x(rkdz0-}O~ph%HkR6voY6r}}3rAa3sC4?ekqbi_u54|Oz^b(p% z1S!%2Qi2+YB-BttLh{=|&pFq)U;{oG|NJzWfgyt7$|YTk2VWOj_+HvsW-W*6hn$Vp75?N3<~EW4 zboN&B*^d$mN3CwwwX7b@N#fl4XdrUzQI&y0a(nLU$RzGZ4Gjmy4t#?8uHs@mWR7wk zQ8ZCF`Tmm4WBCJl`cevvOhfJwY|P60bHB`FT4P@95~f>1b4u|`rTEgN{-BCwJlubC z+$G1}vS`y~88a<)WxpPW#KZr;enIjzS%aMHn7y%tNBc{E@{QR}6I1h!I6pt@c)ZT9AyZ*e)6FR#6QR3n05;-q8aG$TUZLaAHHdL36mL3NqiT=&R zgorS1tmMOpu;=GK9$_ZwXnmw#M0M?5t!dL_muDOvJ9Fpry&z)8XXBaWLLF zpe1x`X+2(j$#5A-URsZ27b|WemR!SShk7+a$$``I{W8|@Yw|7Y*YQliQh>lH1CF?p zSd4$a=DcU!R4SuoNp;Y4uuE#-+FUPmf<^7CvdByoMX&6dlTFq;ANzR{E(g4tvBD~7 zmpFT-KU=fuO}GwAXVp%!is#ptBQD%8J;4sy_?2Rdqy1`x@^Y`9E1?@B)rVH=DpEi?lcA0IE9MV*5_x-+Fx z4I_+JjN>XB8dN>Ky^DNf&m{r}49E@$E`a4qXT9#8>A|8TpA|8;{Uf1&a3>Ffb_n&X zHHO78%(;yBa?zE?4x*9Dc zQE7;#rN+qsYlq;w;Kr*LkqpzaD1pp3z?{F~ zh1%bJnX=Ih=a>twEb3#3NtJSVRRvC6>r+P{{PptCJ7CGG#*@3=Ypolt_a$Gk$uJ4` z*wSKG%FJEVi#k46$h}Oc1qY7bh5EQ=oqB39;JQO0t+C(dGZ-<7OpBLM3UV0Q&*DmT zZOyXwFS3-;)o<MglVErO<5NJeV~Ed?={ z6-N9t?lIyyO@=QDU8z-=KQeoF|LGk6KGoNqs@QIFx(EE|;U)Ouhhu6MS|bLU`PxA0+D-w zV#Xtm(Q9&ce52?-AbfL_iO718YUsqh$znBnAa~_Xi}Fle2|LqCnLKejn#^(gnNnI! zM0yHX*{8-?ai5&{;a3Eh=a2?Ewwdtu1rRR=GFcZDE{+`gvDKf>XJ@9beRcPSrDkSLh^E3MMyn` z*ps*XE+)B*JFuBpQ*sD@ZO}v4Zvkq}{hc{k<|&$V6BTH~WJj%s+r&BqPK z&W~g8gq9i&OMwD&l7;sN0o+i{1a4z@83qe1hug1B8VI$RxuH}l(KwyE#>Z}Gb|`@Y zL=}l=|2~faXrL zSVzAK2C_ZOlC+v$$z%gFoz(Uw+&R#kw)R*xr5#^5e2DucVrEo3p#|LC)h&0#BubTX zI-%ChTu&PueYdr<6D}jdvxRYiyuTT1cx2PTs~*|n@=e6|nPQbISBfgrd>voy3Y*6pAoppi=DvJsp##)F3x zNHMc&wT`{?y<>vag-Ujj*h#mRyReBPr7@G(24;1TM603*9d)+H7b7Xnm**|SXl_i( zUg{qlt*KZuUV{=D2}+9Iql~Nm)1dMqjoRhft&e_xnr9Q33)f3#o@Qc1Ag4NGlCE^GW!RF(Ju(m~y z5(zALwy)v2t$=z13^p+g|R_uKaCHUY& zN6NF9l!U_o9zWqq5g6mtzdv0^J92@hx^OxHW_%8{6=^Q{F-q})rPgf^KR!Heu!K#^6!V=bEY9$M0AR6Dx+UWl1>2#z0256V9x+dOJaXF-vCB1~ci|G4jdAFO=-440ldB zMu5SVGLV#Si$wXIL-qTga}_;>^WAshbj4OEx?9YPj||&i8(mzxzuY9}kl7 z&r~5O9H3o=-Hd~`{qvp6+9pn~Qz<+9*B4rnSqq{@eAYvy6A@b{GjqUl*q)$CUjyad zw(dLTQdLR*!ZCb;l%Zf}$@vd>Fy`5q1!sE$|{VP-bqEVkl;LyyN{Y~0BUw=@O0bU{6L6@26bvJ@u!pE_c+I-d?3 zK9!Zbxm;VsKRkqE0VD#l0+c)-QKf$Og=W2pWEa<%%89NHJX-aY)!_)xZSARoL(k>? zm;5FQ1tKXok9zhpSe6zFmCJ7DPH1T}P?+Isx_5^xqFdl$5dg9GwZA2LXg1)^4Bkc( zzpn`~Ocqb%Q)iWJy!4G6XQkbk`eo^P2SD6)wAhHT?4%o0#gukO7ELx%S+w(n8DmS>smab28JrirJZ<oUjh=F3$^-@b8~6x+Q`F4JeH6GB}3^P zOm});c)x$lQxTlI!OAfNWFbHz1c#xt7iPXTqZxxQAdq&bx@yfOTCm&{W8G9xP$CXM z4ei2e zf#=xYNmF9NxkT{JcC;7PA$@+(-jcW8f`gc0-xxQE-w#{^lCWPnkvxc)#xj4eevvRq z;FJrXMUXLRa|F{;Cajd4pUxUlihVu+={Cu9;Lk6C!kTp&rx?k0DoRzlCUIRpH?7f}OiKE}x)a+H zU@%p6sEma)D8f!E!I_{xMf@S)gU>qQC)7FD$22q(!tWhFj!7%XND;|U0m~TsOOjtS zZ2XfY(l@M4G-OcRRBD4H2 z{lqv{HQ`mgMCk|&Z^1-}FN;J-Psd-?0Wltj&RFU1}+?*YULb%b@< z7iRq8BjY*{YX3>z^K7DDtr?zh9P!7c#oazCbVX>&`T@+PU1nc4)1S$?d5aFUBWgo( zfC-yc@#IrgBGpp&cqiq~{fAqBK8|k?8-;E#|6V<_%PJqApJBCfQbnya>7-;OB%k;?nv#2$_BEXzG(!Z7*jZc5WC>ikXf*p4qlG^ZJ8bP z{BSA*#_Z!0y5hp z)2Xq%&6^eu^$CkjyB9MtN>Fo6ZCoEcckaz|;zG$<618wf$f|zv5P_n?{f7y{BDg#5 zU7D#v3m?D`#C`o(-a-vkP+&o9-IymvkYOaO4(i5$jg9)UeXp*fE4ur#s#&x2;FT}4 zaF-B<2b%C})M5y}n_r)48@KfNN>E!nti+5Ilk_8Fa_c~vCOI=L#Mpl^=`8u4_t#7Q1HoT}hKeJLh!a=VKpr5P58g+A64~ z9!9Tg!pviR(Ud+%vboA_%+awzF4^+^Se4+r8VmZZ6EE65IFEbXQH5lkd6U|xuyt~` z|Cx7OyJJ3gmV;~SvSxc|GCQeit8|j6SbDO)pSUl`v#=!Z@bsVFt9=KRlg7NH9HO3{ zh!xe9XMD3n*8Gks{i+E5ip?^GalGpt29GPa+f4E%*d@QA>R1@8PoXfNQ-th0 zGq)p@4YchjUiu0TbQ}F7Fyc`C&sH3QTGcgd`B@E|`nBm_HA3*Y@!Pk_ zlN=3m0`mS0jnhUhWivq`k}0)~J3Abo0Y>YYviVvrxosxM^X%382@@VcPT#qbsRm57 zyv#&f`|v4;Y=1lUiwW!#7bC-%cY+;o?UT7$MU88soO)|pg0D@!eD5A!goi%uL0fHm zbHd-LNO!WN(ahQLLn0AgJ`;uPiOV@%-63OyVpm_pTKD+NYEd!X zt9|}1Z>GF54I7BG!!sp05;OpLnR{WqKOjriHmW~J_1o^B0yU)Lb1NEHUS9Keyk3#= zepi+B!j4_n=!vt9UE|ew0n@jfN)f{eKK=7)+S5sFfdjUCO;H|G;zoz%)Xlu7DH4+B1 z)Pso5F~CwlIXgVdX0cud(X60JqpY0=hpm2?U#!DumEHwPy6@gXYOWScRJ+}zW(r#3Cf|Bw!3UfF9$b13ae2`St9nx1N+n~PEX8n+2AI`zSrNW^P0Na z`S)|oSLS*%&gW{AurG$xm+JGp$xBmB=i3A|xc(v>C5~{usSHm!3Y3T6m`6~2@-cjC z-rnA()^NMmyE6l09lr|gt&O7*&vZf-zt%)=a;CG=V){Qsc)Ay9uCy_a)1ESMNGkGc z$^S$Gh4a63yt)WAdJ8F|-h3a$!SfI~$fbu#tY2}sqf0|!Aq_XBBG|Gl;+qp;dv$%j zp(rX)wv;F9n%9rH#>6`9F%u!aMEMBoCK+#9uL>s9A#||@>ZuS}*+%Z*lK&h(LQ8W9 zajP|D2;Z46a+&xXve=1qw40D`z$VL<4h@RJD68Y?Cg)JCJNPnk;i2PRS0mL9%4T&J zE$$FN2)|Qcb(3IedF{CFd$`p#KC~nQy`>Ny)6qo{uJnT2SX@@|(ck%HL?}=LYYi^b z-(-BAhj>T@3|>^Q^}ganV+y*=T3k`6^!mOn2dPUm$WEZWBtBBuySWPC-v;jbG-KyA*CA_KB4uv)huYZt3aNpm)D1r(v}~JTBJiq*UPEUvQ$f}3hhajFz1SS*2$}8(J&kD9d>B^4 zb4YLwBvzKRBkWc0G!Q7Pn9M_jRP42OK$j;BTU&jedlAug{>A~{1nH~LT_w4_rqwv% zuS14C={ZXmL(ZYlIz|_yFm}9H0_U~xuix~LEpje122PZrBGZIax?M7oAP?RhS|P;h zsN9R&B&&G*deje5V$G4PuUCS}JjF)3$hjVm>=r%jF8>Pa+N>V3VjX-6m7sa9t*$~c zcLf$RXUh^P%17@(QPjNaf3kPNG@gZ2@W&T^f&JDsOhXST(9abacp=d{Z^bFYT{=PB`f$xl29oa0|t^$CW=aV?Abr z_2T-}GE+X}y^x&^9F9P&*;H%uVb(#U#LaHgt30z(wR=91FgmPe*Gn zJ4|@q4UeXz$XN5<%JhgYX^Wl?veF)0u1O*+^v*Zsa+vo2Si-Yj87R!9aILr+BNujs+!gMcQJBjH&Fezp#tY7uB9 z{QP<3u;_kf55}fedt6+~6;Up{;r{n!hjg>}zXdSaJ}@0(8P=5;vsR2Eu36*IZ3*4X zd#XBjFDBj1{76joeAhAM9PT|}G|$rwKf`XG43y>Q-8e>4o_idgEOA$815Htgd3bJ+ zo|w`Z1!3Yt6#1I`$o!}Z zY|0<>P3wAI5=Hy;wgY<^Lg3mFn42c9je2*B24Sr`o4i*tPc)_99p?`u=s2woXuiW7 zRt8JS*%$t_(Bw*s)k=I6Zd>?Q6D1-{2&dTRmPzl+ZXH;X+^Ah~;4;oiemY8zwRF7r z&u2Xx+30#4LD_-0xVVIL@~GS~HEaZcpN=d{M7PYL3-ps%lAJV%2(t)9p}W_ahBSOe zs3AAU;Mv^=ZOmZAPf{*2yFCCf@ce4n#`z230Dbk-PbV?KosSeX_^wZ_?58BYIttpN zA?ICxw_&k%;;ji2M8Kl)J`gPDeY-@_XvWnxWzkOG8xF;-~75OQpzI805SgopK9 zL~PWp+%l!Z?*eSZNH$X&!FwBGS!x0iD=3?r{1x{8Q(PVDg@~}oijJW}Dm1&%uyrMm zQSPWYAS`<8G!B`ldosLoKf~ZD7?I=C!8uL~CNRJ6zTE{5o=q9o9PRU`tafe9I=ktS zgXt~3@pHjw$=uOE`-F(;Zl^Dd$lc{^$KCZIQ&1&%miwEAm35VxrBoX*K%gl;wLU-e zZt?#xFDBq5FFvA-zjJX}>j&CIt$~YQt7~eMCTiV=3XuUoKj| zN3g;_3C+ss2&h_)qXR7{AYh%5^h7HA)krTr>K9m!55IncVWxeh2Tr8NfNDxz2d!pt zscCuX*-100C&XbT=~e(P?~zKH|3CK-rdz0=up7Yi06NE#wQ1AEX|o#;@;lu*zfRznIb__Ta< zP{9ykb8}O7NTPa|&PE)-(2WtG-|4hqGjfiBjyV72PJm%+A4dt!a6R96Ro`rhWi3>b z%glkB!9aFAgBSOUoyNv?O#~9Ip`ybU(Y0gFq;=7^2mGSLX3gcWWGk1bxgX>MtX6Y9 z@~s-7q8SJTF9U&lz_k^xnOTPT(Y+|mxHk~-rweTz0K6W=Fe)@_B^~^|1IyYWWV|WZ zbfY-@;A?j8T1qJoiAtd!!FJ6JOC^yuI%C3O=41Vdy;k?4sn^S>G7GIYZ>@E+{j(Qf z&>{zbEdUw-LyZcH-0`tOqdDH`Cjfy>%fD6ush`F3*12|o7!n!Q+m(Y0q z88hKwTc6$}5OqT>CYwCt{$?&%%oA0o#6+}O=OZ`)6?1+vQmAzwcerR(1XMb)8&zHH$V z39xoV+;##RmRz>Q^~#f#;sa$g)~z&onxuo&+uSvn9k*bYc{h4ye{<|J+PGb)ZDRi@ zU*!fjMZ*%_ym4TUTJ1sQZ;zo%0_fUig$_6|H6z9h>_UE2>6`vLl4#< z-6kEs-gKqtL3c$5@lw{=msSy{R`@yxd2(4vOtwb2*cGk-)~?ylQs~2s1vRdbw*eak zL20ipXsIl6`ua| z{}!HJFsC~J<9`YN_k2?19(w$J2=u#uksSI~|BS3aHvG?C|Bicf4gU)1|EKW9QR$XG zv1AW-KRA1@|7?cP%YN%7qw{Vrd(!8tfQ{BxaGco*ZE0MokDF*4wcP|d2Dr5WhFkDHA_ zRc77cN!yjf;qJfS@sADCwEz}AI?&zoH2;vHnD2bpm?vaH?P_MxhlyF zY<-t5yt%JE2)}7uehgc7-pMEh^NiTd;Eaz;#DU_qsgHm;xVsh>Nwq#?RgU#lPvEAf{_ zSjr(RlLppxZwD;Y^iVO_ow-z&j~`=Zhi|4>s5mC8l3`gq-gc?=@9N*wZ4 zNr|AqQ4N9(kE1V6TSQGWJXCKsc_!)5I2=tYkXv@}R(lVZ&_f#E;^+@;%R&rgcvYPy zA{W*Q4Fev`5)sReCZa`Nm@l~^Rjy2s4w3j_+S3_-$1Dx9z5Q6I@{61^dlJ$Q58$@H zJ+nZfwU;s6Tj2pC(pIFc=8|JjaU$lMzH5bZJKBy9?k#i-D|}xdV;S>Z#}}c}IEjUu zcJ+xdVNLxZe>LnJYBpbGhBG_sF*IIv@nE|aU^Xcn5b*rCoi-nFmuMUyD*aVU^8q;i zIJ4lT`w(9kNL-?dkGNLGqe6xK9fK|;YPZMDJ<>sb@XSd(zYoZRg9OG~k4FS+-)ZT& zeEKC9iC~xhNp(Y2>Y(zn2c&PqQi#~O*2k^?!&Wkh+e*qaiQv(%8YIhfJdCfxOW;zW z#$;3qOm1^yy^$eKr3^p>1q;sem)-5YR&JD3sSljGF zk8Vm;5vG^bw5?cb{bf)2iPHPYDUT?ls8xd&-qcZ4aqz)Qvc*F#6+QmeeGTf}5fs~I z-k8d1iiVAy;-v&5b(NCI7l$I~>Y-->|NNm_x)Iak2|`ENi+vZV!*^yVKFlp8oewEj zF6+O%_*29^>pgs8ZXjs(ft%<>s)W4fITSc$y}u3&W&}cXFr-#(ySdyyNp#a9HXdb> zeAP8P$xpU77)-beW+U~ap6L-fb+4sX*tB9xNkzpaPTGI%O0NzOZT^vf?%RplPZKej z>+)?Xk_VAMjXwmV|@c4F(zWp71WH_?W2~8?NStNK%HnGW*)kfwv{ZQgG>p&iq$ZNu#j%}&p_{TqFPTs(Z25c zi9;MSw^o-{{I1N-6;^C~ca-m6JB!tYZ<#3*Lsu(UMH z9hKeh#c0(ecG6li?-AQh$J(v%Ge_94DosC{>~~2yb?|(c?)%v!j%0x?t)R7ctb4Y- zCly^BdT55ZCdx1N9R1WAP0@wj4A8p@L&F{+7*fV8J2zT#BTpP%rX@n2btgB zx#4-I5iN>nzE9o$>J*Qi2sY9@?IoSq`XvP+GY^a3O-7OHjw!tLO6XDlZlB?4tHzD0 z+gaY6u9HJ7P$k)d9%N~g?KMHC@+8^+v^KgD0I7cX@{n$b*gbN>ou8BrE+&0y!%dh5 zbm4--6TPgC4~LuISr18_SnAKoIfSkFf}QWIpA%qjEVFmr9#+5~Z(e#YEDFXGZI<*4 zjO-1*cZ*JFVPh0YWrDZ10dAq2y}paE>**K9P!PZ(F! zy!dk6vk2*Vle$}HBD7gGdoy86c&W;zg=~r!AljM)dsq@Gtd%iQ37bS2WR(1qX%fCCM82^6T?1Ou$ONCb2FI_^LCk^nk?|19KYfnMx3EV}mkdWwD)QGW8}5ReJSeI%VFECWUHYqS|YDgz1; zxsM@{y0lF73hCF}Vjx+LZlJ+AV!#;9{4JL(RxDblF+ov9B?0duO!xi0+3|25AslGC zjs5MT+P1m2NM?TT;Vc-HWcdl!tk_3S3Cfo&Vn-gpA}HZgq0246kz~D$3zV~~-;YAP z?Nh~NmcvIlFWBTPuKIz-n}hv#8qqCs&>zB>SK;^@IZS&E3V>~v8lb%0+&&2vOV$X5 zySv&tHysQfJx0?ez0fYXn^*aL<2oEbUD(t5BNcv`!E>*bV%~hI|A-;CV2oh6pm3=& zj}fx#$Oe$13omIYh}hZ_5HtRY-MP|Ht3IIMT>=;r1X87L|LFPvWEpf^1BQ}$*hwjG zg`n;X`^2S8GG8rCw=}&Wf(V6F_aBQBbuNyGcxl~4d}c_=lndF|JLFK#;_2|(k#YAN zPsqJE+ztbv1NtQXz^m9WVO38orK_<-8=ukmaae_JhyaYz%WXPZ%Y22%5|3+8dRY54 zt6$`Fr89HqH0>0*ff-gRMAE|r z-JDneyK2^cwNnT$@%|Mz=}Rk8TyGAZEkd4y81F6qyF3NR_J6EX4QwcV=T{GGFQMs7cyxltA1}POrGH+s5C`3W4XgY*I(q(qkc1qb)aru= zCpRqxU%WPit<9@?b5SZQU@)l)k8W9lwEn5(83A<+aMH4oR6% z+%&`41^6=s%EjKNDmdD>HQ^V8u3Oix=350#$vk>>uO$>wQo4F(+JZ@>mi^JU7Cl2bb9pKRxN(C$d9mIBFdR02Vp%t2;hsjtOvWVDHaYT**fSB=&3fkz*oCyP?C8a@Rpi{sF1^0G z!_5k+%Ev^RA(zFZ6`WqAjpoEx4yRn~TJoonal@04-Zg7%dwdkS(L>hY^pM~3q)c+V zB(8D#E~}{zOr>GV(a9v6Yl>>T)L6)Kh=^-I_d2BWFZGm9H$j#{_@~aQCgWJFgRG`c zc-a=u%O!Zic_-|%9;JVWdW|;zn47h;&ag(lX$C$1?JN3#2gU90PP;H)69xA1gevNb zgB?c*ZD)qBrRkpy5%)WvowIJhbl(SdgFWI2yKf8cy7~)h38S$C`^BPYhj(Hfc9M@+kL&&zOl)c=zX{O z#P)L$AIQji^eehn=#IQ+V*{1P`$)#^h{F^@6!@>JfT9618svhF1 z2weSD)1Gy_FKj0XK1!5R{EsLn+N;h-ITu~hnaJmT4Gi^h%Jq+m_lhk0--DE+3CBnf zK+G(zZ!UMLj8vW>&e1(b@pFmyB>-1AB#;TNXR$`1(JE+xNspdLil`X(4)PS!t6;8W z&H!A4vsxw}+gW!XwJPq2R2#c-bZ{_`y|I-E6*v$?FJ2wm-X79)pC!QLY?jn8yH+ z9?%N|MTTvC)i>~bAJ*!h%uHf>n>W2VU<8bq=0ekYE=tUoiMB0rIR@AsXyawHk!;fB z?N)hUc2f>p*J#3)G1S@}J$^kAF+h=Anj}Q~^RW}RX2Ztac}Sb8^KwLo4B25=4&nF{ zM$%u$BEh0*xHYA@*he4*A*SC7+mA6g99!{(_d08x0J1fo1G{W^HMUYPnlig2 z^3JGjNo}jKfg7bgBM!ZN++H=Y6`hP-%+G zU#(L4aC*4jPdTEWLT?{U%u!K+Iy2-gewz8Pa?rYVf9?%-ywk1j7|O0HXa#yg9Ojoy zyU+@>LPwULi@bvKVLxSm&IYncV2Y1)1kMDk?z1ch-jl&YwE}TdXWCA;dyKYP5 zN$5Z&fdHY;u?oHPK>{OS|BTUtgH(&BykWSM0yb6fInc-~XMZ0;ndM!VI=P!i)_Sfi zwJWiGzS-aV$;-7wBaivQle^!j5?C&=&XgGy7Kp;Bc`yTF*pTYvLYr(#Zcj**5hO0o zv$AWTvc5<9RET2`X=Tkj*Ka9yBTsKHEck2`2Kc|AZg78<6jgm@-^h7|#`G+F>hrao zobt0DqRF#z7y;x*%D~D+{fiO5QQuv&uI%8=JH9qMF0?c9Cpnpl(MUCd0us&%sVVG(bMA8wq*nYf5y?NPm@ zEk0&qK}?cHT^x|$ulI$%FUyPm)KH<|)O|_t=p#cH<{6YI${lWDf6&hM)ydDTszupShd;qmWxDpE-zdB>OuxJI&XHuC zkl8g&z6{rBO5RpuIWrwvlmEU(@_&@iF=BS+fSDlH<0bM1UROtphw;)L?a_O@EU8!Q zo3BdR*KfV9Z|irzG}`#dE8`7u@tj8te*x462)^&P4oD1N7!2XIk!%BN>#k7*&6GG$ zxLw_06@Yh1KHSzi_E6}CP5&Dur+7^!fd1^?BRX{G0p3I1e)BE%`6Gtkft>y06?~ovlV+f7g+8RR1&yF3BJzluQ^&?z}T(!x{gcYfA&MNNfNQ$ZLaC- zX+Wj;yG4&IH%4u(oxAv-V*RzsK$^{lh4{&S&#f@Q_o|5s@L$OrnN2v{T;6?m-sf(~ z2+v`S zbXWO4P6QRkji9LvKhqi(>$ndPch#iqfEN({&D<{d{c9198D>Y<}Y7iHh zD>i#YSWy`ME1}X1Uz$}3YfRgS!&nv7SKt)?-7070dgvJ-SdSfk91Rbnv-|@a27r@2 z-q8n~%w62Ax=~Yg^GF>pP7Is0 zeh9gcWdc2S-_&lZ04`U?LrM)ln3$liYbrp|^DEDr|8{33dxxAy4W1EW?$(c_bW%z=NgmM}Ks(;3Wk$=Ih%U>d zoh;4x!%2P1T{D<>Qjg6z?0d^6)g|!$#r^zXuj5nA3}(W>dv(*^CF0mJWWrn<4S!rq zN$+g~@tr&HCXABfF}jQxznn76@vhN<;_9UeKdK_j{*9?u*8>tRlPI=&FHLLm0|yQ{ zK(+N!#NT`Klr9K2NR8d||KOacEn}!I$uLv$X@+&;8sSBx(@!HmHI~%p3fR~Lmis{y z6H|oOSHh+u=PyQtweK9@%Y}|tKN>4t7a}ch#5bKPy6CvEUjlZw&t*o3Va?#!-LK5p zywipWQ(>Q!n95RGwfar3k0`Pfdt>JQr2kfr6D(oXvGJ| zovPp247%dvlc`_&x=NBu2?k5kKF1}xi@rAI&tMwkQ)(>O0njSVEL1;Yq`7Rm0}@t| zh@kN3e%;T)LtSWC(!4@@H&=H#S3VU3=KxCHvricB7D!;S*TPXPOUA@Jp0FVOi15GC z66{}GX8J7qrScta1hkpkB6;+P+W>YXjZ}>OHXsx!)XLn2_efBOPuP3o)jZOp6h!y+ zLB?c?5M@yzx~|3PFR=zqih5LM`~FbO>Y{~#<;c-Ve{;x20@{f_OR zw%<~ap=?lu{esQcc0JK|3lVV#WQLBj02QYR`IHX z$xWLV%>FW6>&NEj8n(QjsJGI>N$InLXSl^-G3xi8sADn}v}CxZ2qjm--h3d*P2ES{ zc70_tJwr;k>t-B{7U9mi2(hqiOtWzz-7C$*zvW&1U~$_WFg)Z{mTxlUHZUkxzp7C( z+iT#~Ee^B`>5+JE3=dEMy~YB}Q_)AIIDnsjUV&jEeyL9tBcE4&Yu_KY>;pQcDWBan zEdJ}}J;6%-r(TNKHb4U_p)kLN;8udY(4P-3HW^NT59+MVoe}-i#!qmmuU3h>{u#|X(VYN9Xu`|Sf@Xu=}c&4e--1+)oot$d9tVs)E4(KYwQ^wfhIBbDCW z)-Q}3T50#Jd_EuFvT>_xsBW4>22ZQ5mv5Fe`?*%b+)ZBdpQvmvu(e*f<_$JTQn_ED}((l$kaC7&$1iulvi5l_c!V$F1_r+kiVUg`Xa}UDx`~@O5F6MyOqI zZG$u4gRBFw+gvs*gG)F)#FlpU+kOdHm+#OLD)}wKoRmZpir{?ReK*ZO{eKr%1}^XrGRY8FI}G?J;dlrhe$6h37i%Dz9o)|IxrCHlist5i_f!E!*`iS48+lyFb35;R@|3E%>f`pgBX2B+mS+ zXqX7#8@@kus8?R3pLgvwQt{}^iRT~+QhV9U26fLKM1kDc+euuwJ6$x~R?Z9E7Wm?_ z*LDUsF##B5|vPouGw}Ga8|QMN)7NCcfH#wfrTTvko%u zk~|HZ^SqGM4Yxh-S`(l{wV(gEOj|BDDMYUEu`f1xY&Dj{r_a^!?@bK=18zRXMmxB| zj4f5rMsC9qtgVZ#5Cg?$4jVgrVMc#xw=epGSH_?SQH`l1g|>C6TXAIeb>rGbJ#FMu z8&eovtGeP~$uHg)R(a-c;gIKprke&^XcuWp`|v>l)0qw{Nn1s*mDr+tj_R~L5K70I ztY~0=wqjAu3OC?EdoWvM$QTzv`&J#qePMb_P}x4GKz(-c%zP7d;q(1r4K{qbWb5|O zv$u-g(1~RiSQ@qe1C(q{8>$)i?KbGb!?K3@FN%t})a_nL)fcBF6>tqAc2|s;N2Mwo zpN+i6U-V%8srHga^Bo^McQ<@$8~u!4 zZUr#)WK5O(wfLB-eu$3d2xCK}cUh>WgjB*4-u$!%^X+Em^r_lZiBw4q?`P^c1zCwV z3O~TFg&SOaA^fSed(q>EBmlS1g}I2VaRMP?9R;#x0%2St7g4QHa=Pp=4FKs^krj@X zd1Fk0ae--3lpZ+Awbq`j)N0@5@zAb|%tdi?QlFLKDyQ}2wbR%SpK4%%w zx^ zwpL?wsRG8ENUI3i6nDq&27}_h9F{kW_&a7cu#FI7NohqqyAHCwvJKtJn{}~^EXrOb z%>c1+&Cvhgytb?jLw6%RqwYga-i(cMAm{BOoxkmze%38du%RSDM@rPxI)2dZZ#J>p zMR%92$>7@%7n_t@5Mh14l(6?tdVX*dH(K>ycd~A?GznP>G(XCfJ2FBzJ8WM&u=efq zr#cfRQRgAXgDT&(CWAMW)s_uTbKf-ypUw=?t!}f;t=PpInkniXIdDLo`qdWEUAsGy z0ul?*f+TKd?fZYMy?Y?j{rf+TD57*i5h{lclDZ4!uqlNSI*=S!Ne;u3oOhrcQfYE5 zrzts{`~&^{m1q?Jg@WPd0p4@S_u3yD!oH} z5r6B)kHr7JQj94QvaHwafA1wrcls78J(IZ^6Si}T(C3u*b?Vj*$X5vok2G^!7xFIr zE;4ZrI43oebk6w${>RI`Yk;P5qht0;nVYV(K?}U)I zlw;VRPFQ2saFjpb*KeRK9g*zQ!36z^c^WGra{g!xJT@8{P_MhPfPrk*hTe51Qa@HV zJ_8!q6W_>Q3I+jhT6O3XT0BI0=8nI3wb9nzW6$0oRCj;ZQ({VeQoMtGd{b5LvTd8I zDg@opGoN|woE)>%cFQe{Qu`H+`S2vH5srQEM&I85^IHDF*KYX~p~l#WbEJDj_l$SR zk#bCbVeLe%k}QdIuS!(2l^TDQKFV2HNM8K;RXblmh4qivBI2S9#$* ztG1j6Ug-aE1XJ{@ez8*SSZNaxMujiob|L(z17;WVCzRh_x)<>TJMonl#Pdjk9aKM% z?3J@-^`gY(#4o%ce*1l+xvPWkOMdSw>9p~Y#KfxN8LA+KM#W4f~S%@_!!Db0aG#<*dP8K++eFA zN^p5`Zu+%*DeHK6vBwFtY6WzW2ApaSemLU{74uG%{83`k%gHNGd)KvdrBw9CF0X#V zKfY~>uebkHf5q*fZ^`E@g)%x=(N+XK1R}#UX6{6+n)+C33rnJbn{6lGPWEN#I|rz4 zLK?X|?AR}DFt=nrE=3IqD3CGU@@3}dIXBG@9^wkEd4I){0v;%EAIsf=xzu%H&5zea z6Bu*LCZF(Avh5=I&)+i?{>!;&DKhAx${uLsR%Y48Yp5+oa_bx?qbyQbx$R{v??=@Z zm58Xy+48hglL@g|(2Zu|*ymEI@o9VYavJCk?rsmd_aeds=@_D$#&V5fnnTOw9O<7P zIy9Xe=K0+x_9ymX%FZut=`Zc@3f@o~po4l_&AxAPZd)qKeH$>WW+MQA{~7+|!rZ%sCP$1Nx{kXk;3m6{>BnVss#+PVlN%;genBH3 zjTQwrap&i>h_p=k-FiI`6P_yqhAaHXl#>u?y z40nY7Yy|!5ZJq{uF8sf-z>^|c?q6sc$>^~|agg7`YlXVA;hfml=JB1T|B@Ld z7crTi)eWh?ttptBq)WxMM1zBQq1_twdPNk_@;Z>{-wng~+liK3j#EUBAyQflt$|u9 ziR`8;<&Lpd%B_DN2RIa9&j?$9P7CSv(kgy*=7SbHig<#Q-2h*Lo z9}=l+)WeP3Se@WAoy*G^@lxDjQ@s}=2noL&aOGcFskDQ`XCen0pk&m-<%q8{{?kvb z``;D2-|fVrI*SYoF+8q(j~C>p4Q!Z;HAjNMk!*H13Prz4Q2)=eEtndeCfT5Vf$Qoe zG3GZ(;iDY+0<_6|$^cBqWM30vA(iOykv!?$c z=VPtBP&2D!9siLYu}588?*7%^B;=V7cty~`tL)xNA%EJ zy3uf?@vjQ<3Y!7_R?|T)8e-G*Gat0%_Eo z{4{-)ou`?5D@%sHc(j_0xN)&bsZz&_@HaJoT!F_XrX6v?dfK|4cp5LvgKqdD+aSf2 z5S__oJw&69rR`hCdFA+bReptxl~ABdmHjToTubUCV-F%xDk#y4sVg z>UN1&%&If^?&%icm$%mAP**<#0Eq;2T|XP?5?+DNB(ep zC-6GpgsI0pjxfD*ZgP3lqw_j#^_0mVmU&rO=W_ux6wr%*_0nW%C*-|*z&#$HhP&SN znkbZrs8NfqmG2i$5iRBm&}E!UM_eJ{-djM~if=d{S`OdClYq$`6Yji3zmJfX<$S@_ zDuR(j1+g2tR_xb;!DPCf z)`n&9FzXi1EnoDd^;e(r-}3j^K>qU0-MMago?SQ?O7kJb$$A#VNiVQ?N1Q zE*w=>C(j9i=T>l0TNUzdp<80vxSzxn#)f}g(bi9Bmj~Z{4%8C!+>SoEsPw;_lB2eiO3g}s zxx=}MIYAfP6xt09J}u}wgfvQ|lZQCKvzRG<8$m>tSvWa6VwS|Au%;6c7bX4rgm0xe zB7O?$42j>fIGt=K2Xis@e@xQ&dE}1h(`MpK)@HqL z{WFV$g-z@QZ=>Ut@5zeeI%}9|m=|f>$0|BE0ajG6oQGGauj<)TwdlskD0?tC6{ zUaDM3GBDLLyUOo(%41Gv{@k7@$zhw`L~$RVQYuo?t=l7*Yf`rrjp+$@v6^E2)p=pY z%}+(hAAkMpU!wWTZJecVMyDT;_ojm7o+;(Q`1JgNwxv6f(Upo{SeRfeoZrOstDvA# zbBs3H%daIGdXw(+2eqf)!Xx~}unT9Pv{j!X>T;QtQp>m3%|xA3+ZU=~POxASYbGB| zdlBcJF6fb!3Q&ur+3DwKq-On#IWT3XpW@WNtasIz)J?vGPD0Z{ZG}XW+g=eg5*Tx! z_J)u661Xi?_TMW_l!a3*2Tp@U1+>*4R@(RELxTm_S-~cX?!V4;-urS;F*w92Qt=wX zqs5DNWu?nBUiLp`VVI;RB7eSo)>m#o(Kv@TYp&U~8B3pIO%;x}t{K^Rd;KhH#qICu)xE-SBedO*DN?kgL z`4QXQi4eq3dohp)hnI&fd-AaY13^C;=^ogHBJCz5gGAjW2s8J+B_1a{Ng~#z09*Ct z>CFga&E$`t$^w7VYs$Lcy&5;@jdrM$}l%V>+LbxbW48FnLU5`x!KF2fSr zNb1p!@prB`sY>8%^64SS5A{1JMoMv;%dkOvWqVZ*6wojN1W<~EGYW=lZ9@DV>2 z9q3Mk`~z8i-i*#mb5;)>I*LQ!q;nejEAz4&7>GIK)MTgUGk#+%znX;fc%O}U z*D)EoulV7en~4OS*Y4+~otscElJw4zf_-+rzq#>*st0KTyDo3Rd72VMf!yot_VztK za#8k`6lbRR=<>o&g3kZYgieC7y0yeBqZ|(&xYs{bV^Qs&>ipoRkFW+O^F`m#P3-k; z+zG_7B-oYL)%>=W=aQ-TZf~Mmdl6h=J&g5zaZqit=&!vJALPj-Y2rjr{OBT(NeX|- zGUM>j3x|u-?Ja98T;N@(h1}<=l7ff}QEg>M+U$Za(;oBnkWf^O=p_Q~c<{e&#~r8; znZwY?6v;2j3i6Fz3hT?h1|&Y0z20}|JNS-m5LrJI`~^a;=yVVL3ExwzDA4b^^fBuF za0es4RVU}fj35bVoDsZ5n!Z?Ntf~?xlx@@3fA+0H%I26f`r@{}Z99`li!+sAFUs?6 zTY|6jNG~|%Y%VBNw^wE=T%9k;a6e5%%&GkEnuert)r_k0OSkXXSnmnVMU|F5O1hSW z`rNNxgfXQ{jKNf_nmo`yAy)m+RI(@7U3N8X0kwI z`jI8Enf))+@43>LIA)1~AIM^_d*Rr8C$e=yNIFb~8l9**m{+%e<8>>!haamK&6mP0 z|I4f)s}UUInfL?3=wqh`GFqE@^HwOz5%+KZV>XrQx*reh?C-}c+j@C!f9_voO(<5XPfEW|?{ZK&c579EE zw#QOll-l_}E&=U85q<0U`E-hfB}>Sm&l4wmRC>%imtLTSOv>^X59HyM27h1&{YsJw zLoJs~o5#~_Fz~{|2>M)E%~B)JfP~lINdE$Uh)N=8l=p^NlH+_c-2lv>Vf#{pyYfhWUWw~5=J>@EvIdl#$> z8M&weg#VRzbFOIziak!Tf=myj6zU1Z(B&u=pWTB1i?q!)&DYDr&Z!(*5q83@O6_K}065sqV@YE1ziquVVUk0&BMtJ0d#jM{e>$Q806ki*1 zV^W&6?GbwfWjPpB{Eo568;P`y=;W%bq+F8Jhidhf&0Inlv!JT_13NiEnEG7;+7idy zXre2+bUIpnsE5skfdd2S45i;rWPCldPudT`+dKWyFgwXsG~Tmj7R)VK)3cXxRNsw- z(xZ_JMZ1Z!x>~H6!TMaVpyy@1A8qoc<`DXqrZ3W#LzfVLQNL zQ>S+uda*|$sw|sd+e#mgsGepc*d0IFz9KfAPBs{K^O9Rm1opivTVHQibHWaIL(KTQ zjiG`>{o-D;-cR-Zk&MU=TX5KipF49bIrzFo>c$Mm$v%z@sRh_taV{;S8lBW{D2?PP zfkJ;gkwzT@@C<>>f$ZT8h2Kx%@q%@<1El?6@@5O_k(% zHFr?>Og=Ybq}InjqP2m=vk>T-FqiaPQ=C-A*XAg7>FkI2=)QoE^{?eW4io5IcKlWk zQA;zQT)^+8h{D@FolZtGdx}$c$@)<(Ih?DRmNn|1j1JshAG~_%vOi0O&}O+&3}`2; zV+`GYcuO46GvLs5?0cH9?4`{`JD81VqFXlqA_i3VT`+)fXL||O$MAv~z^0h@lFqIn zwbig^+*O4g-ZSaUuKIpr3yp2T_CW3X03Ifk!|q25=*u}xH6mKg?d&oeqA{1!Q9`W2 z(_GukNvak}Zh-@*1_F%@DMO^d*B)D20c(jl*Jv(;=+@+6Hx@UiUva;>h=Z_(igE&- z71!$}UCOs=*^?u5_ON-D?3=nb29l_&pWa){GUo=XiPtFGON7QUOWwU#hDZedP%)_J zeVP~a@}cjOy@|>N>9(AyN`39VCuuOZpx_&6ZtF}x+a>lE7LJ!QXO5i1~Y@DV?!W#K@FpMDO7MfC*K`*L`gmSa z3t4A`5u>PAJDxl~)^6$0cn+)Ed8SNBEc?nsnmAM8*^9{XVAp-j)8AEGC?D0sNuR2R zi-IA2hlI#sk6E@^)c*Hu23|Rs%@{wU@o9;5Zs7V|0DgXB#29%B9!EN<$|#l=P`3VH z=vB;wF3hk`=B!4Z%*=BqZeAkaAZiF1naGO`@RbwOC#3baQ)L?j7J2*4mX4dm_&Mbt zQyykF9!uFF@!o>I?z_HkKD?nXr_Zb+F1|i8{Ku~@*^YGkV1HX@PiMI8Zb{}6Z*81U;(8Q10{BYj;1c9D5^SaNplZcls zO!>5yS7WK;Y|UFRlyd}(EAkyK_G0$W5iDh^z(CWz*U?)x3m^Fo348rh`6~$)XTb;X zS^YjUbnt>NF?b=4w)(Qyozl~Jy!um{?4Rc=t;C&}u3~e7=_b23WncX%zWn8NIUC9s z4P6VIt|s4siMC(0A}T|XEm4%C?^Zum4j%PLl^j7%Mk$9H$oKSbdD9|%$#2uEbB~2+ z2OK{x+bX9LJ@a#Gn?^R^y#q~85TYyd`iOY46=ArxZ`%yoSym|Z4QG(Bqyq;HVI$ux zmE<2;+Uj78wir*-hCjdm-aUM;eDIDvr5*XCHkTeLzOA?{5#J#x=wBupA}z6r&GD++ zR*Mwd3s~MAsO8@DHz_NlnpWI_89~|R^qf`Qh7R2=7sRlf)T`bE@L>MNoED2N#nIR~5d}W)DrX7e0LlM6ZUKt@$W^x0*URuouRnYJ!RAmMWDZ2!W z&IJ6F-seuE7Xs*<#@mV{Oz^^B{Se0e22nqxqwcHl^6o!I8wJ;w==;_Rm5j@$DNgKN z?zc(^mhe8}mIoV6zk2ty>MDabyTSfw0dAn~ zf=a=8wV6C=l57_=&0!s=eY3+;KL<~jnA0s2`mUBWQ))n{R5fA?ePtl>tT|zEp<@4&1cKp~?X=tQjDjIND6}GM}*DI4%>wj(d zVk{JxQZ!%c4}j9HZlufA%ZX~y?-D0U4!wwbYT;O6mp^evz2mN9V>|QBRSQT6N+FShaKQHC`ePD`cq#{Wlt(@AvcGQuRidIH}>Yzf@*5JIlhUwJ+ zLaBPZ1JdQ+`BG1@=|||?%*gA=xAd1G#0AEWQDmZ!#|wtxBFmRbQg-rnyAbX_%8xAx zXBLev!J<#8e=~Juhq`<|70$a^Mmab- z!dh<{1CejVMYxpRSyV1DcywCQkf)oNNU>{XKhOXH4@J zDsiD?_{eL(I_9@7!64s0n^qoQ0b{=(DaQnWZAzTO-6IRlC7~wFzJ(Sr@5bO_Z^r_F z>Key;iBkkU0Uk1xMUVJNgX_gjO=5Tl*nO+tB2#`pLYj$UvS8hBUYUzt^T_W{f1rri z?LM$h_VU^9pkJs`BzUPhjf>|XQ)w-^Lm#|~j=!c{|J##k+<|kd8*9B?`yNYGD%Yzq zu5|xFo*0?!i_zw6i8;>*UL(W?B2=JZdutb6wnvJ7flTRj+n7?RYvT(ZJE$j`e9*me z%aPmLLm53B9aI`V0pROTp#^6t>?zaf!^ApFr)`ci@k4k{?cZt=C&!s!Z09?3^D+LX z)e%gW;MsNvv6BAzsJ)1$hbyEYva6d3GCeH?b$DPbXxdI^sIqtBgSR2_{=F1A!Nd$6 z|1EN5atr`Kza>%j2*0(EeO|M^={?M6WgJq261V;Ro|PFJKp|wM76M!fG^iW=`u|sE z>6i41KE$mYM*Owui0Kk_j#NjTKA*#>y=0;4StXNj^3c=bL$4p4F<7lv!rN`VX_Nlf z{Q-4(Co~_Q98WvBIq`wYC7YA^2Tu1|8f*=gI;o_wuUvWe^XTHOM~d(6cz#@}T0V5^ z8{uNfOW4t%V*2#9h#Oocp_66WD&sg zzjd;f(UYmAKiVC8*ty~h+d@tTlWkOH{%EugY-hog5qj+aelAKZimJgm*C%?xJhnz~u_ z07rl}*>CcrWVhm3qnf|gaJMhdKex)E)WI)ZcB*!)RgBrekxZ5cF1G^7`$;Ba&R@{k zXB0~kQ185E^_d_YUw!iBqTC~|UM6y5mjy$U>e8sN*6P4^Tz~bo`5v1I;$#PHY(o51 zc*Ykf$mU%qjMr5BhB6u-(}yw13&d0e)OV!c$13xUVYM~$^%Icgw%)k{FUC65%@6nn zhMci3JfL}5yIuSUdTD%86c|{i&V^DM%U2yiQ~b<&D%=r{rN-R)gbWh<#Q(A-{xspE z;isMLZW5)bNMe+kK0P^N^X)!gOi=Ubw+AsdT}7As-M*6eo316)DXgv{o3cJqhBE9iuH z2SWlW6!&nICgwY8^>3v~w!2+SRgaiEp6Ekwk6pXUo&_^yTHh>PR~rMlldY)D0TZr; zeTx+3)>agdOkRDkOPO+QSi5?^v6++1KG&O*=X=EPhr#UHDsvfM(_{p#2H-1T#gk7$>}?2r&nE}3W?y&qp>GOC zeKPIqq6LL!`qKNfDTCc5IEvo6t_VN3r;_#6H+yDEXX|w)ky!}q;ZRvEdmUgX$xHS= zu`ZJUuvpvbu8INFlMO@?cOhrcnD_h~l*vgPga0;smRS3tS+iEh_R1Fy%*>ZI1Y>yi z1(H_7t9IN%MCj#XGBrkp7i(%IAuK2V>cudI(w@sGv53-+ML*Qp?n;p727p9Ql_K}C zve}wjbbMvTCul9eD)m)^>Au{)+aCshW>HJb%%s-%=H~f{hYtJ3992r;4uJW{tt4y~&UlhlGwKgKYuwt&@iLDTco-RPEc*LMu=-m8H_eQ2FN- z+k1Y#4P5j)nf^&xZ$C1Xm+*)B=)^s79^8>~uMJMSJp8bchu|~qYpOBQ|2*y9wWEcd z`@{AO;g(Ltq+2DDGE7dgD*Lt4`G-FH~OBS7b>BQ2>rsxZ?3aP-5k3nczw*8h1zhF}$$-#XeOo4XzZSs>k9 zBE#YUrW6%lqsA&J4ukO8H;95u%SCvIJX8^+WQzeyiSJ4}^Uveu{cQ4YjIu_vLo?!o zL^y)?c%+grub55LpJ&!W8=gLJf7G$F+-^r&w){U_oUmK(Rqy7qslnw@GHiZqOlvfS zRkX`o@v{hMyw=9tfHVa3$Cn0wyA zCj4&Gd05J|WlrN5kwUzE{iy%Qs_p9k1XZPlUsPs?f=)86bw}KkI>K1zUjR3{2DWRB zc0Kco_sR*rbo3{AGBzJpojKm{$H((eAg=Hl8!Q{bts!e2{Vs0|7{O@_IYW%*zm40a z_Ob`6f-S4A32<*hrkfQLgh3A`cAb*i&_fB^51&ZHH-@8Jln%Jm>bkl+LblBJs(ZyM zJ>WMZVL->*Rh8g}bEC^tRJ~ zyJ+dNd?QR0aVl(aRY@a75QwO{ZG%{*ze)>ywif^xzil79kpF=!(qrq)A}cmQag zQFp(%AQ9OX42$tvr>dCC$1rv+wBHlf>NxJ=Sb+dHwu3Xvevb?5%Nm_YbV;Ia+Jxwv zGR&w!-<~vdcmC@zoJh~PIhHAW@oX>!>{H(IH`=C$h4gF^ZR=2l8WFMqb?r6dcbOSg(g#4TiCtvGbvgwW@OGqa-wn=@f$(f^S7c@;UG z>n|NBvnH8hFwgCl8WkLV*|b~Ps4tS*^bakd6_?+n{CURUV|}Scbqw&Wf2+Ztg{rr9 zZoxmoY(X-ZC5=T8WXY+|^>1#T8mFD^5pBUX3ivG^0W!pSnECi862Ui%9`cyayvLn2 zw(8rhY~1I93o-tq#As?!Q#PIyV+zKhw&rxYX_~sYqIdwkYY>(5f=*<(SKWGcP(64e zMla6X(ls>$k;Ckj9X;N+CLY}xq1vkV?psB$@g3_UA+P?Cqy}f(7?B>nCw=~KJX($| z`WXFoVX{2icM=Vve{5!>Ub*$3Bsg5jYq$%imndlN;QUZ3G4Pw+@QnYpzNEpWu+LKt z;eKxye-XogmkPcF!#USv{zLf&$aeUd3Mh`t6Br7bB7 zv^f&nCNbkBi2A0vQ!R(%%Xj`f6;z7nAx+19&iugJM=IPV@#0 z7s@1n09O81;fyM(^>5cgAkWkZN{U|Fs3H%0dt17=!=5IYMjr&s(t&rB1_rU zyK@xP=FyOQ_#)6MP1Zj9E`uY(YqwM&{bgGL%{#*k+Sqo&7kj|9l|%!ydqqiq`_Ky+ z@Vl~UzvLms{)DrEHGzP={mZ$4G*k>L46s)Ik@~zi6 zdor=^h$8=`zMf%6zJ$0n&(50Um?*4Ao7@FRR^7)@#t+*KgIWDQ-d>?$ZBqQu0>R*1 zSy8{|*+oZRfj^oQJ6=eNr4S=MQ#>m{#8oF!!YUm^oZiZ}RH#5{sPeCTs^~k>v6)5+ z;oBn>woC>2Pjcf)A89q*=)T&D2aHb*tNxX|olM_D>*urelCIr)14~+v%ISeYs^M;q zGoF&}UdeK{Aa_LuzY66Z)5!diFDI5z=gad+K~t4pQLW>KE5a%hNC#oom-<`wE*Nz7 zbJ1fyC`2ff&mABc8v202^PF{otf*6;XBr3DxF%Zf5vYrv<|rNKU#ooWW>d}VJg-kB zf=oxzbZhVUt^F*Kdj!h6{Tou}$#Wp3Gj`}*M(@cbJ? zvrkk~Hr+_5ygq!5(vdT+917KeCBt@&IJe@%Qwy;b2nB6#V|vZq8xhNZ-1v`@`0KL3 z_nQoLT}RVRpd+m2QfU;sW%2FD5{n_ z=a_prXGo5`2v3>fe3RgXq8sA$8Z9A#T3atzF$7C10@YQY;d6BpWBqKQ@~wml$zc$0 z_VBuHM$sTxm|+uWQ|%5m^73%q%wQ*r-tC1d7eoMhwoT&Q$WZxbb}z`pv9|g=(_#6$ zp^O#T!2gdEeax3gf(PF$Qs!06)3o}O&)DUVe_UiP4*Dy9$F<^vp;^cl)Qk@#c2&S6 z5^(mAVpWFV_JzB0)(b(?1(3gVkDw!(96uZ{44yP+8v6TxIx|Nq-7i@{&!X}u+W^z} zh3YMtvX_}Q>7|XvV0&6wrvo|nG}B=myMCUYc~mz5M#-j@kl+Q=*8Qc`QEN9>JRF>; z?Z%D>`&`Xb=X{=e*e+k=SK|sg^&esqgj!&ZQwm5Dz$iowEy!%YR`;d|d^&`7-Z;GV zma})Rl45{QP%D;Pk$*ZxT()nFjQeAXz#X47As#~|y2!##t*&IB1h{r<#?6x5{2kM& z*ki33y|ns?^Tj>j)_VPIZ#c-?!+Odj70r6C%Liapnu5%-iPdt?;;Bl>RGre1L7erI zKA)U|$r42EM+zg+eXrom4-&K$@4GboXt|>9qpK;?WfB=+9HyXIagtdHnc2_<+mv@h zf?M_{Xael~SWlbO=;a}-Cmh8aQuMxb8lf}gya{J%A@QRcZa@Tuwq{(nPVPA|=CAtI z2ejF<^^NaBU&jb2<9(-FUFG?Ov_WhLw_lb(bz}6|LT+KZn9k_*^OKP$dKR7(B(v;q-&)5QP^kx^ga=pK~lQexpMEhxw`BWOPo%)47fDLyt zLZTMFLK&W5Yp8+5L!fmHC zI3x-Pqy`h(RepA^7?PEBJVGEuGB?j3U8J82Q4&m18c}KseRFk>Gba2Y!#n3HOlL!J zd#F2EE9sb(Ovo>A@|a&2rx}K=*J-p>P#D!W9LZ(fa^x>?=d!%kLE(DOKzW_JRVWZ< z3MTez_1_77zZ>#oCEk6~ z{fGRFc&hbJ_UzCxelv01I(fm*(PMg{3xT_vKUc*{m7Sx#(-6+fh7Xf40%pf$Y7Zz9 z-cOtN%8b8_*ab-z^_(slpC39L8~0e}?njUtDcI^P9{4c`lC zC0%A1A2q~*W9et1f=tUwev3T+*3d~a)Ubq;YVVJb8w-$(Wd zN<3Z~wmK#f+)FHor8rJ+N!G1RUVma6#@nOSF+OaQYs$b1g4kdFs*$E#h4;lGmW440hD5*L2?|MK>CHDCVi@eQa^_N{ZN#AK>&l^fC%pGZFEeQcif5<2cbFzMXy zW;?wwE_!odL`i4Eo2u_+X@TAd4;0ZLSFeX~BN?uX!9C1e_wBZz4>`^ARcSN0iB5|x zoaXVDo-U3~tsxyTIQ6fw!Sy`da(b&xDOr20eG#o}Ph3GVwn5x+B<&2g7gb8Wz;w7v z5A81UObzUEZY41{g=8HV$aq7JbSZLqbCqJ3vwEg-?Kb^Uad8cb87_Mh%RUDrm+~Rjw<={45-4-jNLBy}GkL zvNZG)4^rdnwcMD&@3e; z?nC@q@s+&Et-@XR^Yf4D-;X|TA82*v{>XWgAND5}#*={R z2;@ex-avbAVD^haqg<#hbJ&)fnF9Ck^k z^?iOthy!2KQ%g)EwObt{gFPjRCnHba4oZfZ&X+X1-+6s?dB2+oFE4_4tO@N$y0ePa z#T@xIpS1b|c)|%yNZ`cW1$cD@}<17cKf)#}jF7*dw=FOVrb~u_4O?-CZm#PC$w{zO?Qy830CW_oC~n4h6U{ z^st?Li8X!S$?Wr9@UfXSmx|QzxL2)?{VQl7V}qCG1+d=zoeB5URGL0s=N?^u-hFHO z_Rg!S_7AZBcHfVz$g4S7?^vkR5IRylTsz%40U$EtMM<0+#%E>mvErK$C8xUP!kqE_ zf$fQS!;a9eXkq=^b#0beqy*A8BkiQ+aa-B}tn{#$NK#~Fy zRQ2xC))S1Ez)VJgQZ&^?xq5KWdP-}yKb;R`Yp%_+n|dRvI#ac=+G5VOdwJS}0Ssp5 z$fj01XlGdneK0Qgb%q^~udmPCYqcN<_I6cBDx7YjI|b zuM7-2Y?3(b!GrPJ&)88^PGjYY|9ZQ*+FBHM0H`vuw zl(Rm%;FK<>uN7w*qzokIy`<~cC%6v2Em5QlPESH>+tykEKRA>n<{kO8TxFD^Z1f7F}T0zhd#RRbqUUe zQMq2`wQs6EQPYWG6EPd)e=%03Oxh)}x?guETZ^LUtUtAHf6`f~%Anz_2SE!GyXAyH zdAV}Q*`no(k+S{hJ?`p*^CaO*8{#8kaEBM`a{qA`D?-P#3ZpNOk@}X}!gMqCIc2Qg zQ-CHOntyJvsna@(Gen_x?eR@+5cd>&C=+61PSkH*eeTlxBVO(aH%_(PCi=YpeqBY7 zU!`$B_ViFr?^)hL;uzt&T4~Uk^28mUIWv*pGq2Y^ND)kBm3JUpL%U2WTSI4lz(>GN zn118G?q?*2>-*sO6mjhfsg(C@QU${UmFDNh*Ki-pQ*W<^E)R zNtfJ85sja`KB0E4LhskSh$Yx*v|v`1KG?PMRBx6e4k^ArB}OkLc&>1PNyy+b&pIK0)d`1|qCHRN-nQQ#PN zkS1joZ5FFY@FU-oj4PKL;`F|KhFV{7)@A*(B1D+5lvEYkc8=+grG~&KsKWhNPeCZl zr25Ucwik-uN`I(l?Ps~}`k74Up0suQnsGM3_Ae>AmFCFFP+#W2#}`rWHE;W!5pTSI z-EFY{wq>eLPPo=AEhW4~o9vwZlhnGG_Bwd2qP8YWcfSAFL8r-v>*_69yq^J)SIP;o ztxS&;Mhz$dlk*e{4fn2d{TN{f5_DiOKni!eV29NE^*wwT^ry&<^E1GgBgBfWu5j( zqwAf%DJEn6;uf>;@piglPa=!tv_uJp+U^e0ElaJ2+PT?9*dzMhPDaA_TN1YPmx(Uf zF=_S6v{xrT3Mv-wpUfq_kmhWKxp-VCm-S6qT$;yK9)0Q#wCmMm=wG32|M{~sIrYwi zog=t4MzO0M+OJz(KSRRW#EcAM9wdib-Z-X|pAv3yeNWY~zNeQ|lgi}9tdR4~kITKYfR_fN#@^@ z_B7mMzxA2lYQu}b=5T39Pa)uXG90l)zwbZ#fKm4aaWO7goix^osVLm=Xm-3Z1!(K3 zbl$_+oXp|9fG=Ix)l08Y@*22tDlV9J!K5n-yYJ1fo{(@`+qG-pHM?VVE&>H)V2^p~5-vcF;)>fC zuJ_?`#WlAo-|@_w&3p=wI6OZ3gzTjH6_m^<{OHs#0DC6-{_ZSM1eI|wwe|DNr5IK^;bZWK_B59^e)}f17GbXXP2ks-`A};As~@o0%k0{BcD|r1?YmO1 z&VBo=GUkVNq<9s#m8ZiOZ$ZpjLb+M;xk>TZzHS3f(BXTvTWJbgD+;LKj9nZbmg?Y{ zR=oYPPb-ph$N&w1!(-a5am99Hk5c{C;Bg>wQkY(Ni`wsHahmMnTxhgLA^|Uc=)kil zY@}ONFfV;hURHh72zECN>gj)UvB~ITmh|L-kxc_E0?6gm`X zMOZ9`98HE82VPMlTq+!49Y%n3U$&N=eM%Z+J67JMVy)J0f30m&>R)s8nD?Lo$gve~ zFyFq2NYZN!JwGTMKF;!Il6JTJm8hRE1pv4xYQ~v5;u7rZ-My8Zf-O3Ruh%QRe$uiy zY%$+*3LTq;^Zz`!QT04`e%Ls}@)j}8b}@;6E16GE*l<8DL{*|Qrf%eAjv+@ib5~)O z>GYQjD$i%Re>KI3Q*vei7YO-_@eGhI;dsQFyGu@|ay5ylTMx_H-oSB>w-km=HtT)c znh4a0k|8G!jkm`)B)S;JF-k}7?8q>KpTqBSt(q1YQvA2>JL&Z1wyHnt^O4?{ zBk{p1jgH`DpaDJ^Vwd!i85-SrTm2WmzSF7gkw)n)N4GRo4MlECKx`}TPT0_X#nrAfs+@5%35q)a^A;j zF_K!-cUt^G;=b={orDft{Pkln!JQm)Zcftv$XmDS00S|eH3xELt$MM=;ZWzQtF*YA zoSOC*bS0mtma*yaK;0td zfn$AZJs^mDYSxsV&B?B7LA%}|Yv;`G=8O--o@f1@p;LJxBjJ_JBS{2obJI& z{7I%x$`4vSBLb7M*tMw@koyHc(^WZNqB1T#LL4|z)_~R;Dc94Y`30a>yzaHquPE58 z@hDJI8RbC@Ki1PT2InI5^xR!8P$v7dR2rnz1Ej&1YmJ7SiCXYT2VMQ;+5OyCUme?) z7OG3eI+lwE^xDrAML3S&_raGE9lpY-{d0?+au4FE8I{zm7`3c><1)u1xCBUw$sQ-| z(nKUUAvgF0mx zdoEcV$_^*qsN2 z#zQ-$|H)PX5AI*jj$3bPYyw-Z9z8a>eqHA=#`Z_D|Gi!E!ifz)CqOhcqm)WM%IG`x z#U<^kdp%?LzQ1GT74mQjhr4C)o$nTp@at;;k1q2fW~B)LNL1)awvvn*G*=rQW*qf$ zSE|SQG~wcp(P*SEbr+L4ig;$^l25*b#B5K@r2{1=9=My(0qX@abTY?pf;R#Tq9`VV z3!W;A^!@j6#>>AH?T_XTr}CR>-}Je>K7#kFjKOT>AjOk2}lcw zC<4+6flwk42@rZo2ua|+!Q(lH@NWyZioY$g}4ZL=-X5f4=lIF_nM_#{W#qvQlf&sR z(|Nb{6*Hb18+~p=-~`rYC-RBTg#NwH9xQ4v-K>-a&TRi#V{jN$Re^1OOKp&4yXDv6iRTpg%(rS28AQAB zddIw{X5s@MAZI3MJxJ8%uuP8b-RY(R6NykQMWKfiG_QE7fy=h>Gia!5yh`KUOit5U zmYq9+V$&IKaA*<4?iH?~zY_53UIvouniq$0#${LXW+?RICq^6UnZ6XEHfXe634*wP zx`7yvOVUO9)+EcmO_y4D-{B|PC(Lnig=cXwe#P2paKtXD>I-c zAAW4;dLpxprqp7DS2!9Ug1rh(K22IleU0oc^qn}nxX96x?wKuz{hW8}u3?{aOlb9u zOuo>ETm~efOW>y}7j^I^-3uH7;4nIv4(N`O>D7xAPXi@kV{q~H;tJwen3cTD=yR*V zWn;LRTaN>2LHVxs zjk3~~-hU1D8>iOn(4gc%rw^08@)uBQ9yGwY&^-Y7F|_#+$k2LY+ukO3Jg*M2bzBSK zK-Yjb$@3dx6BAz$+j8r#kNugWyUz-E22Ey);hTP`6W#-{?;%_e z-u%(#)Im*~X{688+wHAZtAg(vbp(_XmgkztqW7=%nJwGEhS`eWl@aa~fEL_6uQ;;n zrihV4Od)-56tw;U&{p$-r%ZLd&@m3#5-*2j5@vF5no|iroET68iO5yw? z{-N!Oy(X4))N;@6H4gIdQCu_*aYa5A7PT~Jd zKq++PU&uWF4^5XThffqNGstcnRbh;R z$XMM78-rXcd3r9YQpCo~-TjwLoW$x5b(SFIW=4V6ELQDZ6?9Y#WU8<6l-Yt(7L55% zy=h9VSbcFEiF#nug!Upjbm`75H8~K1_O_#^*8D`)FBzEJ(}YDF4Gi6JdC~e@41!z? zM2r39HikRvbR-+--xsZQxz{RTbF$HPj!xpV4%pi18H2^FiP3<3JO{-o2UW&$G&QsR2yLO6IbwU-n|zy=zakEZu0SJZx(o0E7vD zS?AAjc;(w)7l-wn-q0Dw1zW1LInVnQH7Ow-$d|SVjE8Pm`H?Z)i02Eqi))pKa^F!< z3Iv8#Gdwil+TlKQK+f??x2DOL2;_^0T;jrCu&NKaiVl3Wha=$V;BrYNK=1#u9dE5~ z@GtyY28hwn!A4i((4KQ*G6k~@7l|dsNskNjsin7C%&@dWj8-28U0b}F$_L?~T2qF{ z+o}&Q>y8YjTq?R4K3RG|qnnB9w*|3$Udi&<-X-Gbf;GGjAZPrytRkxLdon*gQpt}6 zK~||jQ4Ow36ArT$T`T@?#n#>j+u1fGSC98&ZuRBGm|WPHhXWSW-Y)555_)Kde|Q&+*ulgCkoB+wp(%k#d z{~@wxw(VxiIgF&__}c6ikoNweq$<(ES8r4a^V_l1{l#(-f29!P=fADO@c>oQfS869 z213f4YKnb91)>ue9e-?jdGi3ta8?G+dSLc~k1~e{aGfry z3r|8rN}seyDw%LZo}VA@>xTTF>ze}0EIxv5egCX!SQ_M8kiD3QxsvV2IHpjyfp(JF z@~8=OokXeGFQrAMEIPuz9&{bsA$aO!3vO@Lah(+(YoT2UEMGj55Bw%a@Is?8!R%xc zQDUjVuI71&(^8ehhOL7J8|KMNT@@f0-^WG(q`E)$p3U)L7_yIneo6p7-5-(I^_L1m z-ip69{!_p6fJ;A_Pptj@nfN~qJmel5M!pW`>{M$s%+55lO@c{GKjrw%z z>$t?;otk6K z^R4s2BA8iysq>H0Kz48KrZ@C&Ij&qDdMgi=LCSB z#FMnw8pKP?l9X?}3rL{ahd*aC<>k?mGCKaJnOhsT z#jYz~V+-gYPMD^#X?Z%SG&u-J5B+qSu`_)fHO&M1(h@CRp!L*?ldJo(-8tz8M^7+u zS025fc}>x#Jq5QF(&3a+7+Se679g0-fFj!#Q&gXnVcRq7cn8ArnPa&6lUC!AqT$G! zY@5SFKixh$G7Rj|7yrP4the+0(nFnCHtD>{QtdkkqVS7XLsTMeE=y3qg*Y{$R9>ra zo<={K0)Kj^5bv#A8!=ydPjZ_rzP|$JUfyM_jGIWH6&n(tK8TUm9IYiQz%7TO zdF_JpobWBIlWyNrMo!eh`Mp06_i5{j!1W%Oo2e?kGl+7whzdbRYp%MnPYI-R9^gEb zw?`)y1GDhv9q%Q>@$V&8OBI-rOm69FC#f?cLo@7AmI}12Lz7m2yMA%4dEo{sO|VczNxn)hxm0jD+gJS^Ms(qS)aJ@xQr(RtjQ z(C(GL-Es<+#P|abar=y0Z{_}0rjE>L0r z&(z(Lc<6dJ{#n@$GmB|tX6C)9p({ptHREnopgsD20|LRKV}7v~d$&Iktha_@1?oKRDN%r_Zb8k}p z*%1Y2N{O|=rhhdzD;f^e6DplKq~PG~?vQx#T-H*p2hESf}HwkN@qKW4vMkRgt1-84}9ET0I@e{qgl9ccJ8o z8$XY_>;GNBTOCZvalksuqN-k`JEDMW)ZAbV_j&|Oj1X^{evLofAlU$U=9|?`O+NLY z!aeh>eNBo#+7o-=Z{|MTeglBl)!m&p=&uRiC4`BEY zzvaSTU0|Zm13k5~PvF{TXYHS5{^f4%$8*3T`Hz?S_sKv1ns7r^mk@hQ@H*Q1lON>5 zmtX;fn~j^QIvPjs;rJ=I##lGWpPY$bKGE#` zTYd`U1d&%FQ1()MRpY7C$-(w+R^!{Wx9HO8~S8*ek8+Pzth3f)7o^1!zQ?vT>o zj`hh5j|X>8a5OZq0z>~pL_1G@s>`i1xbo`>yoq$;=m~23>dPs*S%uk-uLSr! z=9s$a(RXHg4KK6zj)8@hG+KK-!x+gUwD>?xeKU-PMfaz`M}8(Iu+y3q(iYHxAn!} zeI~4dW!X-Ey1s;z`hQ%(H&aL#7!QZU?HF7Zh->r$zj>)W_6ZFQiUNMBG3iwe{918H zw$<$O;c{COevH1=I;+BV*idi#I4xn@+rFVr56PTH4N^xZ=h-ppE(c%I9FQF+>MO>3{f4TqziG~|u%7|-y5Q5R zyMoyTL5XH0Z!YeR#={`EsDmwhIQyaY?gQ(D6tkts0twn~y&}Bxvyp_jE{8noC5s9T zobm&jhaE0WoCD5VFz}$kc=g+weD%ZSW3C>E-V~Ux+fH2Qv-v9&;iu+<$GSI`Or897 znf)$;kUgFS>Ov0-iq3bw4`!+kl^m6?ealsj1}FFA7hvr+QopN<=J!)JifzJ(?M+F5 zFe!3lDxVEmAu?O|HRZgAn|waKI}xW-+S3ah@1dI>4mk63&CX*C8Z1#%E$oD+*Xt7lmSs>>4{WN6H9$6u0WXD&B}i`KDUYf3KrcvqKS+J3%+ z{Vb}_P4x!qN2(8Kt`by*s>TE&GHu}xRNeQej&}{2tJ8+FFC4!lZt4yrlD*sc`o%+` z!5#T_E$8coh@1Ali@s&Ik1suvJx`WQ2u_!Jm_En7Msez^uvB|zcOZ{5xLNiN-VhyO zr;%B10=l?iTWI%Y{*ilVS(pL3 zDyeqH8C??l<}erw+OR(OVp+b+wlBnuR6%6=hbo-)LIjK1U6{I)BOFC^J0yAx_N(9_ zgSf3Q`QyNrp~JY(!>G7C`GiyYuSF)Hk3kL0SEx1HH7r)))oBus0j;W;fwdDy%VjK7 z584`GHOxc=y^eGGlp%UO%B7&#>mBH|A>YysSDKRbL1^CX_6aU~)GI<$x|qGOf~OzL zc#wK1!lU`f7&FUt-&yoPkkArxh*spTnDYq;4x!#d-&sR7ITf_!5B#(p`#iseSTYYTRrpt|x|>b+fL<(ME`j6uYj4QL$Eg z#bvooxl4Jp@%s{anO4YlRl2)()%3^xmQrUtCN{I$h_Nb-}cP^S*vl(q;0}QpiQOavUMV_F%Xa*kuZSyu zJ!DqLwjOB)AH;zvVjrtgyikh`l!eo`f$;Nva8!=n$h%>Pg6=>YZ3TxG5++L~2lH7q z#NPg#Ao*~@HBGd&9x@RtER%^fO;^9Tp$5(IL_f9_V5^^{8j%6K?juoctg7|+n1KI- zSH_jIZRq&8!X@!ixQCm&PuN-1KOFa=JujNCSzewQz;Cdb+s{Obt5& zBSip}k%M>pf;@Bw!cUAI(d)O`Y2K=Vks#5{uG^9*JNFpG6;eaW&4&rW`Jr|^Pf`mK zUAo0S4k2CmAaRsUSf)TnZ7a8%-!wm7x*xTb-zw59+1!&o_AW0o>fUE%P@ z=Wf0IF@Ir>_K~#FYr@KBGM}k#TuP3O$ZX09l264EQVa-lW~5hK4kJyt(7j+kp?@bE za{TK{OT<_S!dx<6L@V9N#4{?Sjjyu;IGsXdrxk_}xcM zJ;qZ|Z>E?6I~`XScEJkEk{VaQ>860OI+OxmyN2M7nymu3yEA|29q$k;$=n(m`*|j$ zjKGfzDX3a%gL=XsTfo@oq=eZUhOY+Dlk{$r(tV{oSg)m z^y#`qx};SEkl8XblIw$wPGWt5=*N0Ez$CO!sL!k{A8Ssob|4%-09**Pp1a?CP~p2r zQWV5-i)R50|HqqT)=q6w@6})L7VD4a)m7h&Pl-zr2L;;NIVc}%m>;TpiPZ6SS+#Yl zc~&h|#~U|XW_Kum20iHoc7--8X1mFb#lQ6{cOnG$H2ZV1+fQ&T7qK)1fn{r1>+Y1+ zJJ}`Zd7>AV@2unz5&)(OJ&Zd8vhMb&xwrJX^^F%0M%9))Hzvg0_t$7GzwN*L_kvm2 zHzjL!UC7ugOkU{`aQ9WO;(R&?DnNID^z6BmNr#mHyDP^8TQ2us2LInE@DGxDz~trS z{uh_js(pj09Lm_GqBo5tZg}4`2BG9ROl}tyh*Cp{<#)QTb$Mz=`pWJl(Szlz@;R<= zl|WRTjcM{VEsMVIz?rPQ=65|ri&H;$+iu-PiiMB!AcOqjG&lN4!n_H$Zks)k!ZT|(wo1F9)Oky|Vrjc2KjG>P?GvC}d=fpeNXy?FTSd2E7X zevDiCK~|9?$TgsA=8hEp4*g7X>tc@IR!P4tRCoKA?BryJky`657FEt;Z*_X=j?kME zcTF@4b?l2rDqM5cE%h|MoeJmN<$fS=O7+)%0|%;Ok){U6xW89h#gS2bpQT(`u3ml@ z;yZxO9AtPc@C3Ghcu8@(`>Cpy;ucby?eiRaQw~=CZOMfeo1h zz4Z?MZ6%SXiCx~(@Uz)QoaFNr;SQ_4gK|eAP;CeP8!nTcn?Ul9RX(-({5p~mGMQOQ zTll{&^xoyZc*Px*fHgCAvzm&)EJ!hND=d`a*blU^-u$y7i$yU_-;=@C1(jn`Bl2cp z3LIs>p#Obspo^0!2zKZ6&OIAGeuS1AaLHMG@$C1t|DKpDqLD4sG|IQCv+gwpIY-Av zUFW&I{q#~K!{1Tf%I+5%8Leu>WRqs%rMo(!=6X+xpE&sM5Dthl(vVr%H<~IAI=0mw zLCs1;G#?o~`uF=jwiC?Um}Ch_aQgf;gYo5TBKa%(sFhd{PTcqDGpP!7yOnwL4y)QHrD#Jn4Y6eIoJ#0dvr2s6dv6%{ zPt&*D=6R%t_~D?o$`p-(sZD0TScfN$pQ>12UScsE73^Tm9bx+>$r}qTxFV0{$=POQ z+0S59Cq7rxDD*6mL0-tZ(WI^^L-s#>a+It2^?hGXCok*H=pc@@u})>%b)^Mel|LHe zrOwyna;z3mtBeVq!+9@HGq~oeN<=T2KGuJkX>_XV#SQkgpbr|R>uKp}1jRr5wU3vb z{D!?E^a@*u_?6S5hq6p!c5fVXCw3hKG6jqLKeEN&csE_kI=yeiDzj=P7JeAcPwI86 zCTW;e{m zOu$wq8c2x@EI04&KqyNjb+|*5!&vAZ%}te95!Wv3)n(gMWXHf?E}W#Ya{$8e64tA+T&OSLN@!q2TVtar8L%( zIcvwg^c$(~tsKt9p1meS#h{Tj^8&k50>fJOE3$KP?o%d%R+LckA6trB2x3qU`3L<6j(V9d@rI=~!!7z)vgTN-0Xl<{Z$0AqIIuz9pgkrKcFJGx$j<=#Bw86Lk+W2Z+Z-@tKb?R%a-=_Sq zpFrST_O@+;^X{N3ZLlCiIjH2oU2nRY#1YBOfK9u{eArtFO&7ZyP^J(;g;xYc|K-ND zx!gF1pj|Q^eP)0b41D|Q($%-HZ<3V7syJriBOQuD(w|j-A|U+&r)J2CN0|PMxvzUG zju6q3-T!sZ;+-x{6#858-GRsT+tY7mFpA1A{oB5V>T4jP*ppTuq=v1~JyK}1EuzoD z!eU|0^Fg19w+#`0GupJDoPLf8`yE2tDc}CW3NZm2;H`y9FjJB{p*!uGmwRDFv!QYz!w4 z(t^LAa~hIJ72Z(}x_+1QG|+dLtd7j<6-LE&UF_*^G~OhZ+#K^lBdkTFZ|2TWp{zsD z5<=7>b!;GUrAw+gSYPCbn=o7T1!2*rvAJj2qG+jqhQ131m7w+Eoz>n4DH1E2o9)?Oz5OCNr}nMS1e=l^!wz9e#9*&$#mVwJkli~XSc>yr@W znQ8WDkuqt#YI(g&3jCG)llR7>pU2{qi6bOM--Wz>r6Z@OnxWfw2nu{Mu0t1kTxKaY z>02&sG5pGbZ*feAfle_mVfGE{tM)m9{Zf5huu7-ijEy5F&VI6(vGj&{>8}`Jyj1Uq zs7k));^MkNAudt)@i$4IX~Z$K&Y_|cx!1HdJ@c+ZDXM%=+t|9~(RJ9~{-%n2|CPzm zUl-}^14P6*(PraJm0rki2{8G!_t3g+OD}|2ORW??$^Ea5HH#2)MAaZ@-hlyS6^oD-wNse4bBW8~&;S zz*-=b-q8hv8gstKSy@^_G{+{WZPhM=t-3{WoR}3*p;3Cxlh}=v&L&ZXvQp-?P#s{p zWqW*gm+^ovUizQ6SL*N?Q%$7FuDTA5ali#+iPs)v-+MU4CCJF#S z`ispA`O*l~iC(}Fl63y36<}Q>&Nu`J;xDj1#kdsO;DZ>y(f9F3?=)-;hOP?>!24{V zdCUDV%!RJ8Wa!c*0G37%;A<|P$~OhvqHkFS5LHp)A_nNnv3cgQ2qj}pLKB;y30v>bdIgUV8SaZ|PY1e3+y zE^X%Rv5AI=?Vwir0t2QNuvUBy&wjs$Qbi-Sn{xrPK#)h18#0eCk?2uA{aDxwT1T0p z@v3{|)p76M-@FJRk=uGx!vyvB!98os$B}KG+pAM+6!oNFx-`Ju+5Pu`gr3v+W0 zv>D5}>mrcuI#-npAh@tTCK5WF`z%{e1^;>L+Ou`s>grckPtIobA>H?;~{hg z;r?XWu1hZ06JRET&AVH0H-jfVsaB;yK!EcqlF!e2yM=*uas64gJ}?q;JZ{3POLA?E z+-Zem_ivr;H9rCEt(&A#&iwYqudjFdSt$5xS9zO>EjcdG_jg7+2F` zWmn1&kt-J%&|pwcGu`!imnAMeo%#MSt@eXRruW3oe=VKs@HbC_BX<@8J}7@fpdIXF z-GfC#aLpo#~pFcYRSX9iz^l94YTuC z{pTttho~6YqljqCXYM;cAPSqmUG8*6{kFlrTeOEM*bb6zM{fWcow~6OH2B*uRy4@< zg=7VqG*mbIDM0ew#Qpcd%lM>;-u=`Cu$w{pNIM>JHIqVzqkKLlcWxALo1oruC5RNA zHePxn8XUN7b*ai#4mFzM>YrNrTSBWt3gKO5qUe+FL}=6_-gcDwf_|2~TKgzH|2%p8 zZ>vBRN^Zcb#SGrHkPrur_+vw6e5^q)2A*&IDwqAj_iWrkzm!`EHIm_naj95b0w$&6 zu(Pvt{#CbQB4S3st^hea;2Gcbj_UfrNg=v%BS?(n+FOTxEwrI&aCCR;{L>5`kO$TN%- z%Ba>NXu~Wi_+PaLwo4t+kaZVU1Y!|kD)B`zG!k{PF=%HJw=lHi!iu{}gr-JUX^o}J z_X|AG#z_9aB6~N!F0FON1>QoYe1M|))80;79t4rc{+YYP2wbV3mP=Ymw`q4LU{yZY z+7(n=ql$`aLfk)*SUgq+3Lq05_KOA@Z7a`jEQAP~+^&z!ta#QcBdX$(?X} zyixQtSv0Do;whs8;z~)eXLrsth4GO1?Tx^L{k& z1~r3@tTfZl=wUej0Nq(p+yd^$oGL)HZ z)3&R@QN=AK&mwdyBT^ek{?hCG{9%^xciwoZpb33Yik~sqD5?@cd^M&RENzNzEfig} z>PUWBLUj0eI0bBYAdxK#y$~_F!(=k`JcV%%F^djDkn!1U`)$!VnSgBwi(F4M>HW?g z5LNdQ?7lA3ovwqJi#GTw;$ln~h_dqrd098f}83LR&_lNO^3-l{nN%S^{ zRIz1Dy@L4&bXjmEdLzz~x%@z;cc5W8)P`TqWK3>F~0#07eK>KD9i=Kgnm-dw;0Nv?6}&Y9Md} zr6WO1&IRAA;n=Q+d|+q8zrbsX*UFydtcQ$~efV57kHq|pcUO^U>lkKznG!p)G#>WB zl~VG}N{R{@44WwC(_XhbQ_nK=R-+O&0|)v$?UOOV#_GdwaW?NnJm5ILN60Aj?#d*+ z!MWK1H_rR=9`1vX|4y;KZl2*U(K$Vom0n{(F+MnTWuX)vD@xd&iHpOV9u0wZ%snrT z3P{Ceh7d<9KGfYa!vh@BgCdbTrJJFK4O6dprnhvjB8;MP+rWh1*%dSqa4|`NwIrLQ zS_*^3rh3gGw6*$z6(U`7Z_FuFynS~^1k9)_dpVaG`2Fhg)gQg${f)L(cBoA{O#2Ve z#EH_VbF~q@57o#A>9<%MC-fFh5kw~RzNjpI`Z^7ipG5%`fkqDwn`R0O1JJAwSOLuhLY+E!O$3V!~TIOKtI~IdQ_a7Qu)6Obf%nv%xR;cF! zBSF96I?M=KXpX3PNGV_uvbgoYkZDNX5eaAeXHU@gRb_y{^)>HCPup{80G%)tCH#3(O?8X*?4oN`vS6Pq&vP0$g0;!nl$83NcM;-l`KB z^*OfG>^F)nS1~a?4;&7crOLYdi(r*QF=8QB{ys-zIJ0*d;;G{r?Uow1AEoMKx~YV2 zw?59^$az?HF``EwGk4s!a5&Rlr90Pm)R~1|*9M5SZgqf=7s9u;>xjXo{Wq_@T7N*}xuh@PP~^YT(WAcIcNu z-9?{t`Q#UFoGdTy^PTO*lL-5Z3+PAUkOukZ#~U+|cTWlykhq5p!V?G#0rHV2tCD)T zzj!$(z*8)^VvA%g&=c;p6G?8t0cRD>?~IFp5i2DhNp8F4cg5Id?O(~=E(+5{Y_OU_bFP!{H9RKhJ)v(Dzg~c#|y5##`X6 zg+|GcrQ6q%qxVor7u{`TcA+ZyhJ{=49#4s*PVI=tCB7ES$%+5gjuBwIge z=)*je&CpxXaWGKY5b;9RaLj^E$ien8 zJ5GJlG+EtY8unzl+!&$+xTC5ml1VxBj~KzA8vc|CmtBU>{D2?@M}4@M#fMH0hlY`Y zZde(nDRflI_5{Kw2Y?D*PRd1Y%F9HAdv{?ZB&s2W{V=FIi|giGlcqgI@SF+H9mE@S zy~U@jXIo8&3OA%WUJyZJg9?dtS)@z9;%|M%+6`7?hcDO-j&p-}Cc6#Kpc`#j0M9}_ zkCa-1R0aW|hPhw{gkv0gR)&o^4JciBswlqOvn2cX?Uo{}FwLmwYCrRlf7vM_s`GuF zE9WHug!Gd~boF8TjRy-c;6vq+PyJI6gTBe83Ld}STEW5@nziq+Qiq+wKuC!Z=n=!J zScY;28Mz5`UhB#OLq-PrnKNN!Np)Lac0Bx8x93cqop_g^vzpF*z*>Pjp`N683MHr6 z4?7aFI4PC1luI#d=J3EryL`pXPYx@42#XipH5xteQQ-`VxJUwODg!lVfuJJmfhd@D z*s|fh;d)5Fevodj?3HH)DYABP%AW}cehh;3^e~Tq!P29^yZ+6o!-Cny1fRaP^Me6| z&a-_)iyzsx8r>_W#c>LjLGpUY6b(*q3fbUmogldmgw(Weg$)H-jSnt6$}IIa-W6{5`r5K;;_BGE4Sv z;xG5HUXg|2iVTs~*)^?09ax#fd)bdTM-}wlO8XGAodNpSFPGJl3_^^=HMCr&bs-y2 zc(~$(VrN@FxN92J5a(;m2r}oD^VU|LKI)Kg&T3tz*bvbP1+L=7T<&7dI8k z{lHL|oAHLTJu`~fFu#lOu&s%Yseg{E_1rj7#4CXpq8p02CwVzs${ukK#RPqr7H1eWwmQ1TvG}Kf0+%}?u-7no-xu~>7 zGj!r;P{V28CF|$W>h0s*i@K##??pza3M()iWdP=D!(69nABuW34!al%F+^|WsPmE( zT@+o=3{Y%+-`eoAp9mW*ef5Z2^)RpQS%AvFKf%#E*>@_Ta88ujNe*(g>@3t4#a?s4 zw}Dd>eEng1HhF0TrGi-r?l$uR3d@wCen{}#8oZ901j?jGFkbp(Dj3^}admDdf+r~JCYlQFYrl8R`JG0T12KtVAVO0wZP>NIxA6imr`d;rlv8r(t{Upla= z6a9p8+(hsceWOzy=$Wro6QjgD98l%wJEZC;FoV@ z$lRi8ON$ejXAch-eOGaZxCO1S_7}Z7m`F5k4?6r`%&OU;hr|>ez37gTVSXNnNH&o8 z<}jdow_afrg$nRsqWoQ9v!a&PN&2o5-Plt*$|qe2BxdV<<||)&r!}7ipn&?hPevDkTquC?1!mHdp=3Ik z_T_())Ic6|*J~=`4A90YZuh%;G}0a@o0hQ%WXEblR)Vmlc;4S)v5L!|YjeITa5(r(h`xt_`T&;R}5QvlW&<&!{Yn)vmOu(+tTY?JC4j?DluLnx{xVW2y>ew0Bqws#+hv7aF`^ zp1H$8+E||GZ5Id)-bTPfmf%V;K*RJb@I%jTEV;*@xVKrT+Vl)a4W#8-wv9Sv^6>-0 zv4=+=+O;WO0Pwi`Jx>YWR@yj>?_dFl8>F==l~)2}lL5{M9YO$r+R|670uCL6z$I3C z&cC0f!v}z}czL3}5GCM^(2b!(YdmXvV-4uOf}X2kGZorndC7U#WHrr$5IJl%K;1mS z+ZC=>R$K~)^M{YqOg{Y(H$TE+k9Zq0(s?+4sN3{O-_)#_>bBa;@DqiL-&buB^@y5< z#_F0H&|}dc{?weW7Ue!4y7rE;N{U{?H}+EadlN`$q}vvZ?y_U{F6wTFd2O%f>n@45 z&%7BP9{#odyNmXyifr&9^xX#hj@{*?NL(OwX7slgBhKlF6M(6N31z7r3}~xqWsj;`kZaCq#N8s|a!TOSu1%Ur^lG5@fuuuO+|Hspl|Z zVIjQgQTMt{`LL`;04ztu11nJZ5FdZ_ZiO`xD*IGxCLlw>*R^gD z%T+iif2wQ7=KFd)C&_(ubo3$bZ$O$;33=NWlY~~wI2$1LX(e@a@+D4W7T%ku?f}Bv zy?=i?jI*UZpJ_7sxwMq*M+4P@yu3}%kx$vGDc)t-q+&uo#-p}DN<{@}#r!Mo#Jh^H z>6bVKd3kd6m7ZFc!Gc?t-{O4C4YtV!tO^Kio7WLB(iP%)~r)we(6z)Quyq* zgwZ+5%KtW6J*%SPnzfmk7uX5-nMph1mSfaU>z@^xP1RpB2Uu7w%WZunxV@C>TWo!O zea{OE=juf7K<2x!T}YU$W={#3BZKhy^Jn9&nv>@SLw^7Mt=uo2(bd@*Tgi1DxdG?p_XdV%(}6d2;2DQHB!vfuCnrl4KK^OrFrr z?U&M8>a8>uIGeL+wuJ7kfQ(*y*qP{U%X#MXX}_VNq4B7fyl&Zr$y1(VHB0@K54)B} z9@jX`7ur`!QBDZg+MfljY}CcZ#WkN_>zi5gai6G1h(W(sJ%)pK=IlG3-ip-^ZTt2O zb$x$m=v=V@E_O}?2@i4q z-TB^retup~13Z2Qxo-8KYzly-p7RkK)1^gUjvd<2Sf0p-9=(A?s0;g^eY25BfAM(4nI0O z>&t36e>fcxSvF3Q2A%1qDBND0r)~Y5HhL19ALBc_`TX?W={NZ|6cs6Erl$R)ot>Q< zfHt6IbfBdzr8w(76(IT6L_9e!OfzfSYXa~f0>*!ReH~U%P~hj537tc$To_xkcMfWN zHD_~9b!H?uGYOjM+-0KIT`${H$z!Y8Vy&>*a}(S4a&Y#t1sZ%)zRUS z+*_AFW8*cE3s$ughTrf;P lDZ%ag4Pcu8fBoI>RINX`Fzi=w_$vK9kd~1qO#N>7{{sX8+SUL7 diff --git a/g3doc/images/tf_full_color_primary_icon.svg b/g3doc/images/tf_full_color_primary_icon.svg deleted file mode 100644 index 3e7247778..000000000 --- a/g3doc/images/tf_full_color_primary_icon.svg +++ /dev/null @@ -1 +0,0 @@ -FullColorPrimary Icon \ No newline at end of file diff --git a/g3doc/index.md b/g3doc/index.md deleted file mode 100644 index 3b2904b37..000000000 --- a/g3doc/index.md +++ /dev/null @@ -1,570 +0,0 @@ -# ML Metadata - -[ML Metadata (MLMD)](https://github.com/google/ml-metadata) is a library for -recording and retrieving metadata associated with ML developer and data -scientist workflows. MLMD is an integral part of -[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx), but is designed so -that it can be used independently. - -Every run of a production ML pipeline generates metadata containing information -about the various pipeline components, their executions (e.g. training runs), -and resulting artifacts (e.g. trained models). In the event of unexpected -pipeline behavior or errors, this metadata can be leveraged to analyze the -lineage of pipeline components and debug issues. Think of this metadata as the -equivalent of logging in software development. - -MLMD helps you understand and analyze all the interconnected parts of your ML -pipeline instead of analyzing them in isolation and can help you answer -questions about your ML pipeline such as: - -* Which dataset did the model train on? -* What were the hyperparameters used to train the model? -* Which pipeline run created the model? -* Which training run led to this model? -* Which version of TensorFlow created this model? -* When was the failed model pushed? - -## Metadata store - -MLMD registers the following types of metadata in a database called the -**Metadata Store**. - -1. Metadata about the artifacts generated through the components/steps of your - ML pipelines -1. Metadata about the executions of these components/steps -1. Metadata about pipelines and associated lineage information - -The Metadata Store provides APIs to record and retrieve metadata to and from the -storage backend. The storage backend is pluggable and can be extended. MLMD -provides reference implementations for SQLite (which supports in-memory and -disk) and MySQL out of the box. - -This graphic shows a high-level overview of the various components that are part -of MLMD. - -![ML Metadata Overview](images/mlmd_overview.png) - -### Metadata storage backends and store connection configuration - -The `MetadataStore` object receives a connection configuration that corresponds -to the storage backend used. - -* **Fake Database** provides an in-memory DB (using SQLite) for fast - experimentation and local runs. The database is deleted when the store - object is destroyed. - -```python -from ml_metadata import metadata_store -from ml_metadata.proto import metadata_store_pb2 - -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.fake_database.SetInParent() # Sets an empty fake database proto. -store = metadata_store.MetadataStore(connection_config) -``` - -* **SQLite** reads and writes files from disk. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -connection_config.sqlite.connection_mode = 3 # READWRITE_OPENCREATE -store = metadata_store.MetadataStore(connection_config) -``` - -* **MySQL** connects to a MySQL server. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.mysql.host = '...' -connection_config.mysql.port = '...' -connection_config.mysql.database = '...' -connection_config.mysql.user = '...' -connection_config.mysql.password = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -Similarly, when using a MySQL instance with Google CloudSQL -([quickstart](https://cloud.google.com/sql/docs/mysql/quickstart), -[connect-overview](https://cloud.google.com/sql/docs/mysql/connect-overview)), -one could also use SSL option if applicable. - -```python -connection_config.mysql.ssl_options.key = '...' -connection_config.mysql.ssl_options.cert = '...' -connection_config.mysql.ssl_options.ca = '...' -connection_config.mysql.ssl_options.capath = '...' -connection_config.mysql.ssl_options.cipher = '...' -connection_config.mysql.ssl_options.verify_server_cert = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -* **PostgreSQL** connects to a PostgreSQL server. - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.postgresql.host = '...' -connection_config.postgresql.port = '...' -connection_config.postgresql.user = '...' -connection_config.postgresql.password = '...' -connection_config.postgresql.dbname = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -Similarly, when using a PostgreSQL instance with Google CloudSQL -([quickstart](https://cloud.google.com/sql/docs/postgres/quickstart), -[connect-overview](https://cloud.google.com/sql/docs/postgres/connect-overview)), -one could also use SSL option if applicable. - -```python -connection_config.postgresql.ssloption.sslmode = '...' # disable, allow, verify-ca, verify-full, etc. -connection_config.postgresql.ssloption.sslcert = '...' -connection_config.postgresql.ssloption.sslkey = '...' -connection_config.postgresql.ssloption.sslpassword = '...' -connection_config.postgresql.ssloption.sslrootcert = '...' -store = metadata_store.MetadataStore(connection_config) -``` - -## Data model - -The Metadata Store uses the following data model to record and retrieve metadata -from the storage backend. - -* `ArtifactType` describes an artifact's type and its properties that are - stored in the metadata store. You can register these types on-the-fly with - the metadata store in code, or you can load them in the store from a - serialized format. Once you register a type, its definition is available - throughout the lifetime of the store. -* An `Artifact` describes a specific instance of an `ArtifactType`, and its - properties that are written to the metadata store. -* An `ExecutionType` describes a type of component or step in a workflow, and - its runtime parameters. -* An `Execution` is a record of a component run or a step in an ML workflow - and the runtime parameters. An execution can be thought of as an instance of - an `ExecutionType`. Executions are recorded when you run an ML pipeline or - step. -* An `Event` is a record of the relationship between artifacts and executions. - When an execution happens, events record every artifact that was used by the - execution, and every artifact that was produced. These records allow for - lineage tracking throughout a workflow. By looking at all events, MLMD knows - what executions happened and what artifacts were created as a result. MLMD - can then recurse back from any artifact to all of its upstream inputs. -* A `ContextType` describes a type of conceptual group of artifacts and - executions in a workflow, and its structural properties. For example: - projects, pipeline runs, experiments, owners etc. -* A `Context` is an instance of a `ContextType`. It captures the shared - information within the group. For example: project name, changelist commit - id, experiment annotations etc. It has a user-defined unique name within its - `ContextType`. -* An `Attribution` is a record of the relationship between artifacts and - contexts. -* An `Association` is a record of the relationship between executions and - contexts. - -## MLMD Functionality - -Tracking the inputs and outputs of all components/steps in an ML workflow and -their lineage allows ML platforms to enable several important features. The -following list provides a non-exhaustive overview of some of the major benefits. - -* **List all Artifacts of a specific type.** Example: all Models that have - been trained. -* **Load two Artifacts of the same type for comparison.** Example: compare - results from two experiments. -* **Show a DAG of all related executions and their input and output artifacts - of a context.** Example: visualize the workflow of an experiment for - debugging and discovery. -* **Recurse back through all events to see how an artifact was created.** - Examples: see what data went into a model; enforce data retention plans. -* **Identify all artifacts that were created using a given artifact.** - Examples: see all Models trained from a specific dataset; mark models based - upon bad data. -* **Determine if an execution has been run on the same inputs before.** - Example: determine whether a component/step has already completed the same - work and the previous output can just be reused. -* **Record and query context of workflow runs.** Examples: track the owner and - changelist used for a workflow run; group the lineage by experiments; manage - artifacts by projects. -* **Declarative nodes filtering capabilities on properties and 1-hop - neighborhood nodes.** Examples: look for artifacts of a type and under some - pipeline context; return typed artifacts where a given property’s value is - within a range; find previous executions in a context with the same inputs. - -See the -[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) for -an example that shows you how to use the MLMD API and the metadata store to -retrieve lineage information. - -### Integrate ML Metadata into your ML Workflows - -If you are a platform developer interested in integrating MLMD into your system, -use the example workflow below to use the low-level MLMD APIs to track the -execution of a training task. You can also use higher-level Python APIs in -notebook environments to record experiment metadata. - -![ML Metadata Example Flow](images/mlmd_flow.png) - -1) Register artifact types - -```python -# Create ArtifactTypes, e.g., Data and Model -data_type = metadata_store_pb2.ArtifactType() -data_type.name = "DataSet" -data_type.properties["day"] = metadata_store_pb2.INT -data_type.properties["split"] = metadata_store_pb2.STRING -data_type_id = store.put_artifact_type(data_type) - -model_type = metadata_store_pb2.ArtifactType() -model_type.name = "SavedModel" -model_type.properties["version"] = metadata_store_pb2.INT -model_type.properties["name"] = metadata_store_pb2.STRING -model_type_id = store.put_artifact_type(model_type) - -# Query all registered Artifact types. -artifact_types = store.get_artifact_types() -``` - -2) Register execution types for all steps in the ML workflow - -```python -# Create an ExecutionType, e.g., Trainer -trainer_type = metadata_store_pb2.ExecutionType() -trainer_type.name = "Trainer" -trainer_type.properties["state"] = metadata_store_pb2.STRING -trainer_type_id = store.put_execution_type(trainer_type) - -# Query a registered Execution type with the returned id -[registered_type] = store.get_execution_types_by_id([trainer_type_id]) -``` - -3) Create an artifact of DataSet ArtifactType - -```python -# Create an input artifact of type DataSet -data_artifact = metadata_store_pb2.Artifact() -data_artifact.uri = 'path/to/data' -data_artifact.properties["day"].int_value = 1 -data_artifact.properties["split"].string_value = 'train' -data_artifact.type_id = data_type_id -[data_artifact_id] = store.put_artifacts([data_artifact]) - -# Query all registered Artifacts -artifacts = store.get_artifacts() - -# Plus, there are many ways to query the same Artifact -[stored_data_artifact] = store.get_artifacts_by_id([data_artifact_id]) -artifacts_with_uri = store.get_artifacts_by_uri(data_artifact.uri) -artifacts_with_conditions = store.get_artifacts( - list_options=mlmd.ListOptions( - filter_query='uri LIKE "%/data" AND properties.day.int_value > 0')) -``` - -4) Create an execution of the Trainer run - -```python -# Register the Execution of a Trainer run -trainer_run = metadata_store_pb2.Execution() -trainer_run.type_id = trainer_type_id -trainer_run.properties["state"].string_value = "RUNNING" -[run_id] = store.put_executions([trainer_run]) - -# Query all registered Execution -executions = store.get_executions_by_id([run_id]) -# Similarly, the same execution can be queried with conditions. -executions_with_conditions = store.get_executions( - list_options = mlmd.ListOptions( - filter_query='type = "Trainer" AND properties.state.string_value IS NOT NULL')) -``` - -5) Define the input event and read data - -```python -# Define the input event -input_event = metadata_store_pb2.Event() -input_event.artifact_id = data_artifact_id -input_event.execution_id = run_id -input_event.type = metadata_store_pb2.Event.DECLARED_INPUT - -# Record the input event in the metadata store -store.put_events([input_event]) -``` - -6) Declare the output artifact - -```python -# Declare the output artifact of type SavedModel -model_artifact = metadata_store_pb2.Artifact() -model_artifact.uri = 'path/to/model/file' -model_artifact.properties["version"].int_value = 1 -model_artifact.properties["name"].string_value = 'MNIST-v1' -model_artifact.type_id = model_type_id -[model_artifact_id] = store.put_artifacts([model_artifact]) -``` - -7) Record the output event - -```python -# Declare the output event -output_event = metadata_store_pb2.Event() -output_event.artifact_id = model_artifact_id -output_event.execution_id = run_id -output_event.type = metadata_store_pb2.Event.DECLARED_OUTPUT - -# Submit output event to the Metadata Store -store.put_events([output_event]) -``` - -8) Mark the execution as completed - -```python -trainer_run.id = run_id -trainer_run.properties["state"].string_value = "COMPLETED" -store.put_executions([trainer_run]) -``` - -9) Group artifacts and executions under a context using attributions and -assertions artifacts - -```python -# Create a ContextType, e.g., Experiment with a note property -experiment_type = metadata_store_pb2.ContextType() -experiment_type.name = "Experiment" -experiment_type.properties["note"] = metadata_store_pb2.STRING -experiment_type_id = store.put_context_type(experiment_type) - -# Group the model and the trainer run to an experiment. -my_experiment = metadata_store_pb2.Context() -my_experiment.type_id = experiment_type_id -# Give the experiment a name -my_experiment.name = "exp1" -my_experiment.properties["note"].string_value = "My first experiment." -[experiment_id] = store.put_contexts([my_experiment]) - -attribution = metadata_store_pb2.Attribution() -attribution.artifact_id = model_artifact_id -attribution.context_id = experiment_id - -association = metadata_store_pb2.Association() -association.execution_id = run_id -association.context_id = experiment_id - -store.put_attributions_and_associations([attribution], [association]) - -# Query the Artifacts and Executions that are linked to the Context. -experiment_artifacts = store.get_artifacts_by_context(experiment_id) -experiment_executions = store.get_executions_by_context(experiment_id) - -# You can also use neighborhood queries to fetch these artifacts and executions -# with conditions. -experiment_artifacts_with_conditions = store.get_artifacts( - list_options = mlmd.ListOptions( - filter_query=('contexts_a.type = "Experiment" AND contexts_a.name = "exp1"'))) -experiment_executions_with_conditions = store.get_executions( - list_options = mlmd.ListOptions( - filter_query=('contexts_a.id = {}'.format(experiment_id)))) -``` - -## Use MLMD with a remote gRPC server - -You can use MLMD with remote gRPC servers as shown below: - -* Start a server - -```bash -bazel run -c opt --define grpc_no_ares=true //ml_metadata/metadata_store:metadata_store_server -``` - -By default, the server uses a fake in-memory db per request and does not persist -the metadata across calls. It can also be configured with a MLMD -`MetadataStoreServerConfig` to use SQLite files or MySQL instances. The config -can be stored in a text protobuf file and passed to the binary with -`--metadata_store_server_config_file=path_to_the_config_file`. - -An example `MetadataStoreServerConfig` file in text protobuf format: - -```textpb -connection_config { - sqlite { - filename_uri: '/tmp/test_db' - connection_mode: READWRITE_OPENCREATE - } -} -``` - -* Create the client stub and use it in Python - -```python -from grpc import insecure_channel -from ml_metadata.proto import metadata_store_pb2 -from ml_metadata.proto import metadata_store_service_pb2 -from ml_metadata.proto import metadata_store_service_pb2_grpc - -channel = insecure_channel('localhost:8080') -stub = metadata_store_service_pb2_grpc.MetadataStoreServiceStub(channel) -``` - -* Use MLMD with RPC calls - -```python -# Create ArtifactTypes, e.g., Data and Model -data_type = metadata_store_pb2.ArtifactType() -data_type.name = "DataSet" -data_type.properties["day"] = metadata_store_pb2.INT -data_type.properties["split"] = metadata_store_pb2.STRING - -request = metadata_store_service_pb2.PutArtifactTypeRequest() -request.all_fields_match = True -request.artifact_type.CopyFrom(data_type) -stub.PutArtifactType(request) - -model_type = metadata_store_pb2.ArtifactType() -model_type.name = "SavedModel" -model_type.properties["version"] = metadata_store_pb2.INT -model_type.properties["name"] = metadata_store_pb2.STRING - -request.artifact_type.CopyFrom(model_type) -stub.PutArtifactType(request) -``` - -## Upgrade the MLMD library - -When using a new MLMD release or your own build with an existing MLMD database, -there may be changes to the database schema. Unless a breaking change is -explicitly mentioned in the release note, all MLMD database schema changes are -transparent for the MLMD API users. If there is a breaking change notice, then -old databases can still be upgraded to use the new MLMD library. - -When the MLMD library connects to the database, it compares the expected schema -version of the MLMD library (`library_version`) with the schema version -(`db_version`) recorded in the given database. By default, MLMD will check the -compatibility and raise errors when the versions are incompatible. - -* If `library_version` is compatible with `db_version`, nothing happens. -* If `library_version` is newer than `db_version`, and auto-migration is not - enabled, then MLMD raises a failed precondition error with the following - message: - - ``` - MLMD database version $db_version is older than library version - $library_version. Schema migration is disabled. Please upgrade the - database then use the library version; or switch to a older library - version to use the current database. - ``` - -* If `library_version` is older than `db_version`, by default MLMD library - returns errors to prevent any data loss. In this case, you should upgrade - the library version before using that database. - -### Upgrade the database schema - -MLMD provides utilities to upgrade the database version. - -For example, when connecting to a backend with a Python library: - -```python {highlight="range:enable,True"} -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -store = metadata_store.MetadataStore(connection_config, - enable_upgrade_migration=True) -``` - -Or when using gRPC server, set the MetadataStoreServerConfig as follows: - -```python -connection_config { - ... -} -migration_options { - enable_upgrade_migration: true -} -``` - -MLMD then evolves the database by executing a series of migration scripts. If -the backend supports DDL queries within a transaction (e.g., SQLite), MLMD runs -the steps together within a single transaction, and the transaction is -rolled-back when an error occurs. The migration script is provided together with -any schema-change commit and verified through testing. - -!!! Note - The migration DDLs in MySQL are not transactional. When using MySQL, there - should only be a single connection with the upgrade migration enabled to use the - old database. Take a backup of the database before upgrading to prevent - potential data losses. - -### Downgrade the database schema - -A misconfiguration in the deployment of MLMD may cause an accidental upgrade, -e.g., when you tries out a new version of the library and accidentally connect -to the production instance of MLMD and upgrade the database. To recover from -these situations, MLMD provides a downgrade feature. During connection, if the -migration options specify the `downgrade_to_schema_version`, MLMD will run a -downgrade transaction to revert the schema version and migrate the data, then -terminate the connection. Once the downgrade is done, use the older version of -the library to connect to the database. - -For example: - -```python -connection_config = metadata_store_pb2.ConnectionConfig() -connection_config.sqlite.filename_uri = '...' -metadata_store.downgrade_schema(connection_config, - downgrade_to_schema_version = 0) -``` - -!!! Note - When downgrading, MLMD prevents data loss as much as possible. However, - newer schema versions might be inherently more expressive and hence a downgrade - can introduce data loss. When using backends that do not support DDL - transactions (e.g., MySQL), the database should be backed up before downgrading - and the downgrade script should be the only MLMD connection to the database. - -The list of `schema_version` used in MLMD releases are: - -ml-metadata (MLMD) | schema_version ------------------- | -------------- -1.16.0 | 10 -1.15.0 | 10 -1.14.0 | 10 -1.13.1 | 10 -1.13.0 | 10 -1.12.0 | 10 -1.11.0 | 10 -1.10.0 | 8 -1.9.0 | 8 -1.8.0 | 8 -1.7.0 | 8 -1.6.0 | 7 -1.5.0 | 7 -1.4.0 | 7 -1.3.0 | 7 -1.2.0 | 7 -1.1.0 | 7 -1.0.0 | 6 -0.30.0 | 6 -0.29.0 | 6 -0.28.0 | 6 -0.27.0 | 6 -0.26.0 | 6 -0.25.1 | 6 -0.24.0 | 5 -0.23.0 | 5 -0.22.1 | 5 -0.21.2 | 4 -0.15.2 | 4 -0.14.0 | 4 -0.13.2 | 0 - -## Resources - -The MLMD library has a high-level API that you can readily use with your ML -pipelines. See the -[MLMD API documentation](api/mlmd.md) -for more details. - -Check out -[MLMD Declarative Nodes Filtering](https://github.com/google/ml-metadata/blob/v1.2.0/ml_metadata/proto/metadata_store.proto#L708-L786) -to learn how to use MLMD declarative nodes filtering capabilities on properties -and 1-hop neighborhood nodes. - -Also check out the -[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) to -learn how to use MLMD to trace the lineage of your pipeline components. diff --git a/g3doc/javascripts/mathjax.js b/g3doc/javascripts/mathjax.js deleted file mode 100644 index 0be88e041..000000000 --- a/g3doc/javascripts/mathjax.js +++ /dev/null @@ -1,19 +0,0 @@ -window.MathJax = { - tex: { - inlineMath: [["\\(", "\\)"]], - displayMath: [["\\[", "\\]"]], - processEscapes: true, - processEnvironments: true - }, - options: { - ignoreHtmlClass: ".*|", - processHtmlClass: "arithmatex" - } -}; - -document$.subscribe(() => { - MathJax.startup.output.clearCache() - MathJax.typesetClear() - MathJax.texReset() - MathJax.typesetPromise() -}) diff --git a/g3doc/stylesheets/extra.css b/g3doc/stylesheets/extra.css deleted file mode 100644 index beaf9694e..000000000 --- a/g3doc/stylesheets/extra.css +++ /dev/null @@ -1,19 +0,0 @@ -:root { - --md-primary-fg-color: #FFA800; - --md-primary-fg-color--light: #CCCCCC; - --md-primary-fg-color--dark: #425066; -} - -.video-wrapper { - max-width: 240px; - display: flex; - flex-direction: row; -} -.video-wrapper > iframe { - width: 100%; - aspect-ratio: 16 / 9; -} - -p img{ - background: white; -} diff --git a/mkdocs.yml b/mkdocs.yml index 399112061..f9a4ba2c0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,8 +2,6 @@ site_name: "ML Metadata" repo_name: "ML Metadata" repo_url: https://github.com/google/ml-metadata -docs_dir: g3doc - theme: name: material palette: @@ -110,11 +108,11 @@ nav: - Tutorial: https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/ - API: - mlmd: - - Overview: api/mlmd/ + - Overview: api/mlmd/index.md - mlmd: api/mlmd/mlmd.md - mlmd.errors: - - Overview: api/mlmd.errors/ + - Overview: api/mlmd.errors/index.md - mlmd.errors: api/mlmd.errors/mlmd.errors.md - mlmd.proto: - - Overview: api/mlmd.proto/ + - Overview: api/mlmd.proto/index.md - mlmd.proto: api/mlmd.proto/mlmd.proto.md From 7f63761e93d4b42286fe1fb86e9717a0c288d152 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:22:16 -0700 Subject: [PATCH 27/31] Remove `--verbose` flag --- .github/workflows/cd-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index eaeb2a2bc..6587ba2b7 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -44,5 +44,5 @@ jobs: mkdocs gh-deploy --force - name: Build docs to check for errors - run: mkdocs build --verbose + run: mkdocs build if: (github.event_name == 'pull_request') From 4c2ead17c395ae134ebe5bb4eac3fd73833f45eb Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:48:04 -0700 Subject: [PATCH 28/31] Fix triggers --- .github/workflows/cd-docs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index 6587ba2b7..02c0cf675 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -2,8 +2,6 @@ name: deploy-docs on: workflow_dispatch: push: - branches: - - master pull_request: permissions: contents: write From 040369bc9f44fce6ad50caadb07f094f960fd1fe Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:50:30 -0700 Subject: [PATCH 29/31] Fix link --- docs/index.md | 570 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 docs/index.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..43e18a1ce --- /dev/null +++ b/docs/index.md @@ -0,0 +1,570 @@ +# ML Metadata + +[ML Metadata (MLMD)](https://github.com/google/ml-metadata) is a library for +recording and retrieving metadata associated with ML developer and data +scientist workflows. MLMD is an integral part of +[TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx), but is designed so +that it can be used independently. + +Every run of a production ML pipeline generates metadata containing information +about the various pipeline components, their executions (e.g. training runs), +and resulting artifacts (e.g. trained models). In the event of unexpected +pipeline behavior or errors, this metadata can be leveraged to analyze the +lineage of pipeline components and debug issues. Think of this metadata as the +equivalent of logging in software development. + +MLMD helps you understand and analyze all the interconnected parts of your ML +pipeline instead of analyzing them in isolation and can help you answer +questions about your ML pipeline such as: + +* Which dataset did the model train on? +* What were the hyperparameters used to train the model? +* Which pipeline run created the model? +* Which training run led to this model? +* Which version of TensorFlow created this model? +* When was the failed model pushed? + +## Metadata store + +MLMD registers the following types of metadata in a database called the +**Metadata Store**. + +1. Metadata about the artifacts generated through the components/steps of your + ML pipelines +1. Metadata about the executions of these components/steps +1. Metadata about pipelines and associated lineage information + +The Metadata Store provides APIs to record and retrieve metadata to and from the +storage backend. The storage backend is pluggable and can be extended. MLMD +provides reference implementations for SQLite (which supports in-memory and +disk) and MySQL out of the box. + +This graphic shows a high-level overview of the various components that are part +of MLMD. + +![ML Metadata Overview](images/mlmd_overview.png) + +### Metadata storage backends and store connection configuration + +The `MetadataStore` object receives a connection configuration that corresponds +to the storage backend used. + +* **Fake Database** provides an in-memory DB (using SQLite) for fast + experimentation and local runs. The database is deleted when the store + object is destroyed. + +```python +from ml_metadata import metadata_store +from ml_metadata.proto import metadata_store_pb2 + +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.fake_database.SetInParent() # Sets an empty fake database proto. +store = metadata_store.MetadataStore(connection_config) +``` + +* **SQLite** reads and writes files from disk. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +connection_config.sqlite.connection_mode = 3 # READWRITE_OPENCREATE +store = metadata_store.MetadataStore(connection_config) +``` + +* **MySQL** connects to a MySQL server. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.mysql.host = '...' +connection_config.mysql.port = '...' +connection_config.mysql.database = '...' +connection_config.mysql.user = '...' +connection_config.mysql.password = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +Similarly, when using a MySQL instance with Google CloudSQL +([quickstart](https://cloud.google.com/sql/docs/mysql/quickstart), +[connect-overview](https://cloud.google.com/sql/docs/mysql/connect-overview)), +one could also use SSL option if applicable. + +```python +connection_config.mysql.ssl_options.key = '...' +connection_config.mysql.ssl_options.cert = '...' +connection_config.mysql.ssl_options.ca = '...' +connection_config.mysql.ssl_options.capath = '...' +connection_config.mysql.ssl_options.cipher = '...' +connection_config.mysql.ssl_options.verify_server_cert = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +* **PostgreSQL** connects to a PostgreSQL server. + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.postgresql.host = '...' +connection_config.postgresql.port = '...' +connection_config.postgresql.user = '...' +connection_config.postgresql.password = '...' +connection_config.postgresql.dbname = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +Similarly, when using a PostgreSQL instance with Google CloudSQL +([quickstart](https://cloud.google.com/sql/docs/postgres/quickstart), +[connect-overview](https://cloud.google.com/sql/docs/postgres/connect-overview)), +one could also use SSL option if applicable. + +```python +connection_config.postgresql.ssloption.sslmode = '...' # disable, allow, verify-ca, verify-full, etc. +connection_config.postgresql.ssloption.sslcert = '...' +connection_config.postgresql.ssloption.sslkey = '...' +connection_config.postgresql.ssloption.sslpassword = '...' +connection_config.postgresql.ssloption.sslrootcert = '...' +store = metadata_store.MetadataStore(connection_config) +``` + +## Data model + +The Metadata Store uses the following data model to record and retrieve metadata +from the storage backend. + +* `ArtifactType` describes an artifact's type and its properties that are + stored in the metadata store. You can register these types on-the-fly with + the metadata store in code, or you can load them in the store from a + serialized format. Once you register a type, its definition is available + throughout the lifetime of the store. +* An `Artifact` describes a specific instance of an `ArtifactType`, and its + properties that are written to the metadata store. +* An `ExecutionType` describes a type of component or step in a workflow, and + its runtime parameters. +* An `Execution` is a record of a component run or a step in an ML workflow + and the runtime parameters. An execution can be thought of as an instance of + an `ExecutionType`. Executions are recorded when you run an ML pipeline or + step. +* An `Event` is a record of the relationship between artifacts and executions. + When an execution happens, events record every artifact that was used by the + execution, and every artifact that was produced. These records allow for + lineage tracking throughout a workflow. By looking at all events, MLMD knows + what executions happened and what artifacts were created as a result. MLMD + can then recurse back from any artifact to all of its upstream inputs. +* A `ContextType` describes a type of conceptual group of artifacts and + executions in a workflow, and its structural properties. For example: + projects, pipeline runs, experiments, owners etc. +* A `Context` is an instance of a `ContextType`. It captures the shared + information within the group. For example: project name, changelist commit + id, experiment annotations etc. It has a user-defined unique name within its + `ContextType`. +* An `Attribution` is a record of the relationship between artifacts and + contexts. +* An `Association` is a record of the relationship between executions and + contexts. + +## MLMD Functionality + +Tracking the inputs and outputs of all components/steps in an ML workflow and +their lineage allows ML platforms to enable several important features. The +following list provides a non-exhaustive overview of some of the major benefits. + +* **List all Artifacts of a specific type.** Example: all Models that have + been trained. +* **Load two Artifacts of the same type for comparison.** Example: compare + results from two experiments. +* **Show a DAG of all related executions and their input and output artifacts + of a context.** Example: visualize the workflow of an experiment for + debugging and discovery. +* **Recurse back through all events to see how an artifact was created.** + Examples: see what data went into a model; enforce data retention plans. +* **Identify all artifacts that were created using a given artifact.** + Examples: see all Models trained from a specific dataset; mark models based + upon bad data. +* **Determine if an execution has been run on the same inputs before.** + Example: determine whether a component/step has already completed the same + work and the previous output can just be reused. +* **Record and query context of workflow runs.** Examples: track the owner and + changelist used for a workflow run; group the lineage by experiments; manage + artifacts by projects. +* **Declarative nodes filtering capabilities on properties and 1-hop + neighborhood nodes.** Examples: look for artifacts of a type and under some + pipeline context; return typed artifacts where a given property’s value is + within a range; find previous executions in a context with the same inputs. + +See the +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) for +an example that shows you how to use the MLMD API and the metadata store to +retrieve lineage information. + +### Integrate ML Metadata into your ML Workflows + +If you are a platform developer interested in integrating MLMD into your system, +use the example workflow below to use the low-level MLMD APIs to track the +execution of a training task. You can also use higher-level Python APIs in +notebook environments to record experiment metadata. + +![ML Metadata Example Flow](images/mlmd_flow.png) + +1) Register artifact types + +```python +# Create ArtifactTypes, e.g., Data and Model +data_type = metadata_store_pb2.ArtifactType() +data_type.name = "DataSet" +data_type.properties["day"] = metadata_store_pb2.INT +data_type.properties["split"] = metadata_store_pb2.STRING +data_type_id = store.put_artifact_type(data_type) + +model_type = metadata_store_pb2.ArtifactType() +model_type.name = "SavedModel" +model_type.properties["version"] = metadata_store_pb2.INT +model_type.properties["name"] = metadata_store_pb2.STRING +model_type_id = store.put_artifact_type(model_type) + +# Query all registered Artifact types. +artifact_types = store.get_artifact_types() +``` + +2) Register execution types for all steps in the ML workflow + +```python +# Create an ExecutionType, e.g., Trainer +trainer_type = metadata_store_pb2.ExecutionType() +trainer_type.name = "Trainer" +trainer_type.properties["state"] = metadata_store_pb2.STRING +trainer_type_id = store.put_execution_type(trainer_type) + +# Query a registered Execution type with the returned id +[registered_type] = store.get_execution_types_by_id([trainer_type_id]) +``` + +3) Create an artifact of DataSet ArtifactType + +```python +# Create an input artifact of type DataSet +data_artifact = metadata_store_pb2.Artifact() +data_artifact.uri = 'path/to/data' +data_artifact.properties["day"].int_value = 1 +data_artifact.properties["split"].string_value = 'train' +data_artifact.type_id = data_type_id +[data_artifact_id] = store.put_artifacts([data_artifact]) + +# Query all registered Artifacts +artifacts = store.get_artifacts() + +# Plus, there are many ways to query the same Artifact +[stored_data_artifact] = store.get_artifacts_by_id([data_artifact_id]) +artifacts_with_uri = store.get_artifacts_by_uri(data_artifact.uri) +artifacts_with_conditions = store.get_artifacts( + list_options=mlmd.ListOptions( + filter_query='uri LIKE "%/data" AND properties.day.int_value > 0')) +``` + +4) Create an execution of the Trainer run + +```python +# Register the Execution of a Trainer run +trainer_run = metadata_store_pb2.Execution() +trainer_run.type_id = trainer_type_id +trainer_run.properties["state"].string_value = "RUNNING" +[run_id] = store.put_executions([trainer_run]) + +# Query all registered Execution +executions = store.get_executions_by_id([run_id]) +# Similarly, the same execution can be queried with conditions. +executions_with_conditions = store.get_executions( + list_options = mlmd.ListOptions( + filter_query='type = "Trainer" AND properties.state.string_value IS NOT NULL')) +``` + +5) Define the input event and read data + +```python +# Define the input event +input_event = metadata_store_pb2.Event() +input_event.artifact_id = data_artifact_id +input_event.execution_id = run_id +input_event.type = metadata_store_pb2.Event.DECLARED_INPUT + +# Record the input event in the metadata store +store.put_events([input_event]) +``` + +6) Declare the output artifact + +```python +# Declare the output artifact of type SavedModel +model_artifact = metadata_store_pb2.Artifact() +model_artifact.uri = 'path/to/model/file' +model_artifact.properties["version"].int_value = 1 +model_artifact.properties["name"].string_value = 'MNIST-v1' +model_artifact.type_id = model_type_id +[model_artifact_id] = store.put_artifacts([model_artifact]) +``` + +7) Record the output event + +```python +# Declare the output event +output_event = metadata_store_pb2.Event() +output_event.artifact_id = model_artifact_id +output_event.execution_id = run_id +output_event.type = metadata_store_pb2.Event.DECLARED_OUTPUT + +# Submit output event to the Metadata Store +store.put_events([output_event]) +``` + +8) Mark the execution as completed + +```python +trainer_run.id = run_id +trainer_run.properties["state"].string_value = "COMPLETED" +store.put_executions([trainer_run]) +``` + +9) Group artifacts and executions under a context using attributions and +assertions artifacts + +```python +# Create a ContextType, e.g., Experiment with a note property +experiment_type = metadata_store_pb2.ContextType() +experiment_type.name = "Experiment" +experiment_type.properties["note"] = metadata_store_pb2.STRING +experiment_type_id = store.put_context_type(experiment_type) + +# Group the model and the trainer run to an experiment. +my_experiment = metadata_store_pb2.Context() +my_experiment.type_id = experiment_type_id +# Give the experiment a name +my_experiment.name = "exp1" +my_experiment.properties["note"].string_value = "My first experiment." +[experiment_id] = store.put_contexts([my_experiment]) + +attribution = metadata_store_pb2.Attribution() +attribution.artifact_id = model_artifact_id +attribution.context_id = experiment_id + +association = metadata_store_pb2.Association() +association.execution_id = run_id +association.context_id = experiment_id + +store.put_attributions_and_associations([attribution], [association]) + +# Query the Artifacts and Executions that are linked to the Context. +experiment_artifacts = store.get_artifacts_by_context(experiment_id) +experiment_executions = store.get_executions_by_context(experiment_id) + +# You can also use neighborhood queries to fetch these artifacts and executions +# with conditions. +experiment_artifacts_with_conditions = store.get_artifacts( + list_options = mlmd.ListOptions( + filter_query=('contexts_a.type = "Experiment" AND contexts_a.name = "exp1"'))) +experiment_executions_with_conditions = store.get_executions( + list_options = mlmd.ListOptions( + filter_query=('contexts_a.id = {}'.format(experiment_id)))) +``` + +## Use MLMD with a remote gRPC server + +You can use MLMD with remote gRPC servers as shown below: + +* Start a server + +```bash +bazel run -c opt --define grpc_no_ares=true //ml_metadata/metadata_store:metadata_store_server +``` + +By default, the server uses a fake in-memory db per request and does not persist +the metadata across calls. It can also be configured with a MLMD +`MetadataStoreServerConfig` to use SQLite files or MySQL instances. The config +can be stored in a text protobuf file and passed to the binary with +`--metadata_store_server_config_file=path_to_the_config_file`. + +An example `MetadataStoreServerConfig` file in text protobuf format: + +```textpb +connection_config { + sqlite { + filename_uri: '/tmp/test_db' + connection_mode: READWRITE_OPENCREATE + } +} +``` + +* Create the client stub and use it in Python + +```python +from grpc import insecure_channel +from ml_metadata.proto import metadata_store_pb2 +from ml_metadata.proto import metadata_store_service_pb2 +from ml_metadata.proto import metadata_store_service_pb2_grpc + +channel = insecure_channel('localhost:8080') +stub = metadata_store_service_pb2_grpc.MetadataStoreServiceStub(channel) +``` + +* Use MLMD with RPC calls + +```python +# Create ArtifactTypes, e.g., Data and Model +data_type = metadata_store_pb2.ArtifactType() +data_type.name = "DataSet" +data_type.properties["day"] = metadata_store_pb2.INT +data_type.properties["split"] = metadata_store_pb2.STRING + +request = metadata_store_service_pb2.PutArtifactTypeRequest() +request.all_fields_match = True +request.artifact_type.CopyFrom(data_type) +stub.PutArtifactType(request) + +model_type = metadata_store_pb2.ArtifactType() +model_type.name = "SavedModel" +model_type.properties["version"] = metadata_store_pb2.INT +model_type.properties["name"] = metadata_store_pb2.STRING + +request.artifact_type.CopyFrom(model_type) +stub.PutArtifactType(request) +``` + +## Upgrade the MLMD library + +When using a new MLMD release or your own build with an existing MLMD database, +there may be changes to the database schema. Unless a breaking change is +explicitly mentioned in the release note, all MLMD database schema changes are +transparent for the MLMD API users. If there is a breaking change notice, then +old databases can still be upgraded to use the new MLMD library. + +When the MLMD library connects to the database, it compares the expected schema +version of the MLMD library (`library_version`) with the schema version +(`db_version`) recorded in the given database. By default, MLMD will check the +compatibility and raise errors when the versions are incompatible. + +* If `library_version` is compatible with `db_version`, nothing happens. +* If `library_version` is newer than `db_version`, and auto-migration is not + enabled, then MLMD raises a failed precondition error with the following + message: + + ``` + MLMD database version $db_version is older than library version + $library_version. Schema migration is disabled. Please upgrade the + database then use the library version; or switch to a older library + version to use the current database. + ``` + +* If `library_version` is older than `db_version`, by default MLMD library + returns errors to prevent any data loss. In this case, you should upgrade + the library version before using that database. + +### Upgrade the database schema + +MLMD provides utilities to upgrade the database version. + +For example, when connecting to a backend with a Python library: + +```python {highlight="range:enable,True"} +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +store = metadata_store.MetadataStore(connection_config, + enable_upgrade_migration=True) +``` + +Or when using gRPC server, set the MetadataStoreServerConfig as follows: + +```python +connection_config { + ... +} +migration_options { + enable_upgrade_migration: true +} +``` + +MLMD then evolves the database by executing a series of migration scripts. If +the backend supports DDL queries within a transaction (e.g., SQLite), MLMD runs +the steps together within a single transaction, and the transaction is +rolled-back when an error occurs. The migration script is provided together with +any schema-change commit and verified through testing. + +!!! Note + The migration DDLs in MySQL are not transactional. When using MySQL, there + should only be a single connection with the upgrade migration enabled to use the + old database. Take a backup of the database before upgrading to prevent + potential data losses. + +### Downgrade the database schema + +A misconfiguration in the deployment of MLMD may cause an accidental upgrade, +e.g., when you tries out a new version of the library and accidentally connect +to the production instance of MLMD and upgrade the database. To recover from +these situations, MLMD provides a downgrade feature. During connection, if the +migration options specify the `downgrade_to_schema_version`, MLMD will run a +downgrade transaction to revert the schema version and migrate the data, then +terminate the connection. Once the downgrade is done, use the older version of +the library to connect to the database. + +For example: + +```python +connection_config = metadata_store_pb2.ConnectionConfig() +connection_config.sqlite.filename_uri = '...' +metadata_store.downgrade_schema(connection_config, + downgrade_to_schema_version = 0) +``` + +!!! Note + When downgrading, MLMD prevents data loss as much as possible. However, + newer schema versions might be inherently more expressive and hence a downgrade + can introduce data loss. When using backends that do not support DDL + transactions (e.g., MySQL), the database should be backed up before downgrading + and the downgrade script should be the only MLMD connection to the database. + +The list of `schema_version` used in MLMD releases are: + +ml-metadata (MLMD) | schema_version +------------------ | -------------- +1.16.0 | 10 +1.15.0 | 10 +1.14.0 | 10 +1.13.1 | 10 +1.13.0 | 10 +1.12.0 | 10 +1.11.0 | 10 +1.10.0 | 8 +1.9.0 | 8 +1.8.0 | 8 +1.7.0 | 8 +1.6.0 | 7 +1.5.0 | 7 +1.4.0 | 7 +1.3.0 | 7 +1.2.0 | 7 +1.1.0 | 7 +1.0.0 | 6 +0.30.0 | 6 +0.29.0 | 6 +0.28.0 | 6 +0.27.0 | 6 +0.26.0 | 6 +0.25.1 | 6 +0.24.0 | 5 +0.23.0 | 5 +0.22.1 | 5 +0.21.2 | 4 +0.15.2 | 4 +0.14.0 | 4 +0.13.2 | 0 + +## Resources + +The MLMD library has a high-level API that you can readily use with your ML +pipelines. See the +[MLMD API documentation](api/mlmd/) +for more details. + +Check out +[MLMD Declarative Nodes Filtering](https://github.com/google/ml-metadata/blob/v1.2.0/ml_metadata/proto/metadata_store.proto#L708-L786) +to learn how to use MLMD declarative nodes filtering capabilities on properties +and 1-hop neighborhood nodes. + +Also check out the +[MLMD tutorial](https://tensorflow.github.io/tfx/tutorials/mlmd/mlmd_tutorial/) to +learn how to use MLMD to trace the lineage of your pipeline components. diff --git a/setup.py b/setup.py index 0bb2f9f0d..2739ed918 100644 --- a/setup.py +++ b/setup.py @@ -172,7 +172,7 @@ def run(self): extras_require={ 'lint': ['pre-commit'], # TODO: Pin versions for docs - "docs": docs_reqs + "docs": docs_reqs, }, python_requires='>=3.9,<4', packages=find_packages(), From 45257006caa2929f7c71d0c5e913d4379a5da432 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:56:39 -0700 Subject: [PATCH 30/31] Only trigger on push to master --- .github/workflows/cd-docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index 02c0cf675..cc529ce4c 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -2,6 +2,8 @@ name: deploy-docs on: workflow_dispatch: push: + branches: + - 'master' pull_request: permissions: contents: write From aea4660c3822dd70d66d7275bc3ea51941d62211 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:21:17 -0700 Subject: [PATCH 31/31] Fix linting errors --- .gitignore | 1 - ml_metadata/proto/__init__.py | 22 +++++++++++----------- setup.py | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 63e111c79..2b8352bc0 100644 --- a/.gitignore +++ b/.gitignore @@ -177,4 +177,3 @@ cython_debug/ # Bazel build files bazel-* - diff --git a/ml_metadata/proto/__init__.py b/ml_metadata/proto/__init__.py index b1eb7b4b9..a2b7cb2e1 100644 --- a/ml_metadata/proto/__init__.py +++ b/ml_metadata/proto/__init__.py @@ -14,29 +14,29 @@ """ML Metadata proto module.""" # Connection configurations for different deployment. -from ml_metadata.proto.metadata_store_pb2 import ( - ConnectionConfig, - MetadataStoreClientConfig, +from ml_metadata.proto import ( + metadata_store_service_pb2, + metadata_store_service_pb2_grpc, ) # ML Metadata core data model concepts. from ml_metadata.proto.metadata_store_pb2 import ( Artifact, - Execution, + ArtifactType, + Association, + Attribution, + ConnectionConfig, Context, + ContextType, Event, - Attribution, - Association, - ParentContext, - ArtifactType, + Execution, ExecutionType, - ContextType, FakeDatabaseConfig, + MetadataStoreClientConfig, MySQLDatabaseConfig, + ParentContext, SqliteMetadataSourceConfig, ) -from ml_metadata.proto import metadata_store_service_pb2 -from ml_metadata.proto import metadata_store_service_pb2_grpc del metadata_store_service_pb2 del metadata_store_service_pb2_grpc diff --git a/setup.py b/setup.py index 2739ed918..5b1ad7415 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ def run(self): _LONG_DESCRIPTION = fp.read() # Get documentation build requirements -with open("requirements-docs.txt", "r") as fp: +with open("requirements-docs.txt") as fp: docs_reqs = fp.readlines() docs_reqs = [req.replace("\n", "") for req in docs_reqs]