From 719011b2f54f64a736853bd9e7631c698494b546 Mon Sep 17 00:00:00 2001 From: AdamMinton <53316110+AdamMinton@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:26:35 -0400 Subject: [PATCH] feat: new import destinations (#91) * feat: Allow import to Embed Groups and Users folders (#56) * chore: added python 3.9 testing * chore: flake8 cleanup --- .github/workflows/python-ci.yml | 1 + README.md | 9 ++++++ looker_deployer/commands/deploy_content.py | 34 +++++++++++++++------- tests/test_deploy_content.py | 12 ++++---- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 2c268db..45decc9 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -32,6 +32,7 @@ jobs: - '3.6' - '3.7' - '3.8' + - '3.9' steps: - name: Repo Checkout diff --git a/README.md b/README.md index 1f9e387..e618f5a 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,12 @@ optional arguments: Looks to deploy ``` +**Import base folder** +Support has been added for importing contenting to base folders other than Shared. There are a few important notes to go along with that: +- Users, Embed Users, and Embed Groups folders must exist before running the import, ldeploy cannot create them. +- Users, Embed Users, and Embed Groups folders must be uniquely named. +- "Shared" is the default base folder if no other is specified using "--target-folder" + ### Examples: - `ldeploy content export --env dev --folders 1 --local-target ./foo/bar/` <- exports the Shared folder (id 1) and all sub-folders to the @@ -186,6 +192,9 @@ optional arguments: `Shared/Public` and all sub-folders to the prod instance in the `Shared/FromDev/Public` folder. - `ldeploy content import --env prod --dashboards ./foo/bar/Shared/Public/Dashboard_1.json ./foo/bar/Shared/Restricted/Dashboard_2.json` <- deploys `Dashboard1` and `Dashboard2` to their respective folders in the prod instance +- `ldeploy content import --env prod --folders ./dev/Users --recursive --target-folder Users` <- deploys every piece of content in `dev/Users` and all sub-folders to the prod instance in the `Users` folder +- `ldeploy content import --env prod --folders "./dev/Embed Users" --recursive --target-folder "Embed Users"` <- deploys every piece of content in `dev/Embed Users` and all sub-folders to the prod instance in the `Embed Groups` folder +- `ldeploy content import --env prod --folders "./dev/Embed Groups" --recursive --target-folder "Embed Groups"` <- deploys every piece of content in `dev/Embed Groups` and all sub-folders to the prod instance in the `Embed Groups` folder ## Board Deployment diff --git a/looker_deployer/commands/deploy_content.py b/looker_deployer/commands/deploy_content.py index 6f4951f..77d8122 100644 --- a/looker_deployer/commands/deploy_content.py +++ b/looker_deployer/commands/deploy_content.py @@ -33,6 +33,13 @@ def get_space_ids_from_name(space_name, parent_id, sdk): if (space_name == "Shared" and parent_id == "0"): return ["1"] + elif (space_name == "Embed Groups" and parent_id == "0"): + return sdk.search_spaces(name=space_name, parent_id=None)[0].id + elif (space_name == "Users" and parent_id == "0"): + return sdk.search_spaces(name=space_name, parent_id=None)[0].id + elif (space_name == "Embed Users" and parent_id == "0"): + return sdk.search_spaces(name=space_name, parent_id=None)[0].id + logger.debug("space info", extra={"space_name": space_name, "parent_id": parent_id}) space_list = sdk.search_spaces(name=space_name, parent_id=parent_id) id_list = [i.id for i in space_list] @@ -49,6 +56,9 @@ def create_or_return_space(space_name, parent_id, sdk): if len(target_id) > 1: logger.error("More than one Space found with that parent/name", extra={"space_ids": target_id}) raise e + elif (parent_id == '2' and len(target_id) == 0): + logger.warning("Cannot create folder in Users. Add the User first, then import their content", extra={"folder": space_name, "target_id": len(target_id)}) + raise e else: logger.warning("No folders found. Creating folder now") new_space = models.CreateSpace(name=space_name, parent_id=parent_id) @@ -139,7 +149,7 @@ def build_spaces(spaces, sdk): return id_tracker[0] -def deploy_space(s, sdk, env, ini, recursive, debug=False): +def deploy_space(s, sdk, env, ini, recursive, debug=False, target_base=None): logger.debug("working folder", extra={"working_folder": s}) @@ -151,7 +161,7 @@ def deploy_space(s, sdk, env, ini, recursive, debug=False): logger.debug("files to process", extra={"looks": look_files, "dashboards": dash_files}) # cut down directory to looker-specific paths - a, b, c = s.partition("Shared") # Hard coded to Shared for now TODO: Change this! + a, b, c = s.partition(target_base) c = c.rpartition(os.sep)[0] logger.debug("partition components", extra={"a": a, "b": b, "c": c}) @@ -192,17 +202,17 @@ def deploy_space(s, sdk, env, ini, recursive, debug=False): if recursive and space_children: logger.info("Attemting Recursion of children folders", extra={"children_folders": space_children}) for child in space_children: - deploy_space(child, sdk, env, ini, recursive, debug) + deploy_space(child, sdk, env, ini, recursive, debug, target_base) else: logger.info("No Recursion specified or empty child list", extra={"children_folders": space_children}) -def deploy_content(content_type, content, sdk, env, ini, debug=False): +def deploy_content(content_type, content, sdk, env, ini, debug=False, target_base=None): # extract directory path dirs = content.rpartition(os.sep)[0] + os.sep # cut down directory to looker-specific paths - a, b, c = dirs.partition("Shared") # Hard coded to Shared for now TODO: Change this! + a, b, c = dirs.partition(target_base) c = c.rpartition(os.sep)[0] # strip trailing slash # turn into a list of spaces to process @@ -215,7 +225,7 @@ def deploy_content(content_type, content, sdk, env, ini, debug=False): def send_content( - sdk, env, ini, target_folder=None, spaces=None, dashboards=None, looks=None, recursive=False, debug=False + sdk, env, ini, target_folder=None, spaces=None, dashboards=None, looks=None, recursive=False, debug=False, target_base=None ): if spaces: @@ -233,10 +243,10 @@ def send_content( # copy the source space directory tree to target space override shutil.copytree(s, updated_space) # kick off the job from the new space - deploy_space(updated_space, sdk, env, ini, recursive, debug) + deploy_space(updated_space, sdk, env, ini, recursive, debug, target_base) # If no target space override, kick off job normally else: - deploy_space(s, sdk, env, ini, recursive, debug) + deploy_space(s, sdk, env, ini, recursive, debug, target_base) if dashboards: logger.debug("Deploying dashboards", extra={"dashboards": dashboards}) for dash in dashboards: @@ -285,11 +295,12 @@ def main(args): logger.debug("ini file", extra={"ini": args.ini}) if args.target_folder: - # Force any target space override to start from Shared - assert args.target_folder.startswith("Shared"), "Target Space MUST begin with 'Shared'" # Make sure trailing sep is in place if not args.target_folder.endswith(os.sep): args.target_folder += os.sep + args.target_base = args.target_folder.split('/')[0] + else: + args.target_base = 'Shared' sdk = get_client(args.ini, args.env) send_content( @@ -301,5 +312,6 @@ def main(args): args.dashboards, args.looks, args.recursive, - args.debug + args.debug, + args.target_base ) diff --git a/tests/test_deploy_content.py b/tests/test_deploy_content.py index 93d232d..28eb3da 100644 --- a/tests/test_deploy_content.py +++ b/tests/test_deploy_content.py @@ -82,7 +82,7 @@ def test_create_or_return_space_none_found(mocker): sdk.create_space.return_value = models.Space(name="Foo", parent_id="1", id="42") deploy_content.get_space_ids_from_name.return_value = [] - target_id = deploy_content.create_or_return_space("foo", "2", sdk) + target_id = deploy_content.create_or_return_space("foo", "5", sdk) assert target_id == "42" @@ -221,7 +221,7 @@ def test_deploy_space_build_call(mocker): mocker.patch("looker_deployer.commands.deploy_content.build_spaces") mocker.patch("looker_deployer.commands.deploy_content.import_content") - deploy_content.deploy_space("Foo/Shared/Bar/", "sdk", "env", "ini", False, False) + deploy_content.deploy_space("Foo/Shared/Bar/", "sdk", "env", "ini", False, False, "Shared") deploy_content.build_spaces.assert_called_with(["Shared", "Bar"], "sdk") @@ -239,7 +239,7 @@ def test_deploy_space_look_call(mocker): deploy_content.build_spaces.return_value = "42" mocker.patch("looker_deployer.commands.deploy_content.import_content") - deploy_content.deploy_space("Foo/Shared/Bar", "sdk", "env", "ini", False, False) + deploy_content.deploy_space("Foo/Shared/Bar", "sdk", "env", "ini", False, False, "Shared") deploy_content.import_content.assert_called_once_with("look", "Foo/Shared/Bar/Look_test", "42", "env", "ini", False) @@ -257,7 +257,7 @@ def test_deploy_space_dashboard_call(mocker): deploy_content.build_spaces.return_value = "42" mocker.patch("looker_deployer.commands.deploy_content.import_content") - deploy_content.deploy_space("Foo/Shared/Bar", "sdk", "env", "ini", False, False) + deploy_content.deploy_space("Foo/Shared/Bar", "sdk", "env", "ini", False, False, "Shared") deploy_content.import_content.assert_called_once_with( "dashboard", "Foo/Shared/Bar/Dashboard_test", @@ -272,7 +272,7 @@ def test_deploy_content_build_call(mocker): mocker.patch("looker_deployer.commands.deploy_content.build_spaces") mocker.patch("looker_deployer.commands.deploy_content.import_content") - deploy_content.deploy_content("look", "Foo/Shared/Bar/Baz/Dashboard_test.json", "sdk", "env", "ini", False) + deploy_content.deploy_content("look", "Foo/Shared/Bar/Baz/Dashboard_test.json", "sdk", "env", "ini", False, "Shared") deploy_content.build_spaces.assert_called_with(["Shared", "Bar", "Baz"], "sdk") @@ -281,5 +281,5 @@ def test_deploy_content_import_content_call(mocker): deploy_content.build_spaces.return_value = "42" mocker.patch("looker_deployer.commands.deploy_content.import_content") - deploy_content.deploy_content("look", "Foo/Shared/Bar/Look_test.json", "sdk", "env", "ini", False) + deploy_content.deploy_content("look", "Foo/Shared/Bar/Look_test.json", "sdk", "env", "ini", False, "Shared") deploy_content.import_content.assert_called_with("look", "Foo/Shared/Bar/Look_test.json", "42", "env", "ini", False)