diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 99676c668d4d..46eddbffc4af 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1692,13 +1692,8 @@ def get_related(self, obj): res.update( dict( hosts=self.reverse('api:inventory_hosts_list', kwargs={'pk': obj.pk}), - groups=self.reverse('api:inventory_groups_list', kwargs={'pk': obj.pk}), - root_groups=self.reverse('api:inventory_root_groups_list', kwargs={'pk': obj.pk}), variable_data=self.reverse('api:inventory_variable_data', kwargs={'pk': obj.pk}), script=self.reverse('api:inventory_script_view', kwargs={'pk': obj.pk}), - tree=self.reverse('api:inventory_tree_view', kwargs={'pk': obj.pk}), - inventory_sources=self.reverse('api:inventory_inventory_sources_list', kwargs={'pk': obj.pk}), - update_inventory_sources=self.reverse('api:inventory_inventory_sources_update', kwargs={'pk': obj.pk}), activity_stream=self.reverse('api:inventory_activity_stream_list', kwargs={'pk': obj.pk}), job_templates=self.reverse('api:inventory_job_template_list', kwargs={'pk': obj.pk}), ad_hoc_commands=self.reverse('api:inventory_ad_hoc_commands_list', kwargs={'pk': obj.pk}), @@ -1709,8 +1704,17 @@ def get_related(self, obj): labels=self.reverse('api:inventory_label_list', kwargs={'pk': obj.pk}), ) ) + if obj.kind in ('', 'constructed'): + # links not relevant for the "old" smart inventory + res['groups'] = self.reverse('api:inventory_groups_list', kwargs={'pk': obj.pk}) + res['root_groups'] = self.reverse('api:inventory_root_groups_list', kwargs={'pk': obj.pk}) + res['update_inventory_sources'] = self.reverse('api:inventory_inventory_sources_update', kwargs={'pk': obj.pk}) + res['inventory_sources'] = self.reverse('api:inventory_inventory_sources_list', kwargs={'pk': obj.pk}) + res['tree'] = self.reverse('api:inventory_tree_view', kwargs={'pk': obj.pk}) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) + if obj.kind == 'constructed': + res['source_inventories'] = self.reverse('api:inventory_source_inventories', kwargs={'pk': obj.pk}) return res def to_representation(self, obj): diff --git a/awx/api/urls/inventory.py b/awx/api/urls/inventory.py index 7e2fa4ebe2ba..c7d3592c93fb 100644 --- a/awx/api/urls/inventory.py +++ b/awx/api/urls/inventory.py @@ -7,6 +7,7 @@ InventoryList, InventoryDetail, InventoryActivityStreamList, + InventorySourceInventoriesList, InventoryJobTemplateList, InventoryAccessList, InventoryObjectRolesList, @@ -37,6 +38,7 @@ re_path(r'^(?P[0-9]+)/script/$', InventoryScriptView.as_view(), name='inventory_script_view'), re_path(r'^(?P[0-9]+)/tree/$', InventoryTreeView.as_view(), name='inventory_tree_view'), re_path(r'^(?P[0-9]+)/inventory_sources/$', InventoryInventorySourcesList.as_view(), name='inventory_inventory_sources_list'), + re_path(r'^(?P[0-9]+)/source_inventories/$', InventorySourceInventoriesList.as_view(), name='inventory_source_inventories'), re_path(r'^(?P[0-9]+)/update_inventory_sources/$', InventoryInventorySourcesUpdate.as_view(), name='inventory_inventory_sources_update'), re_path(r'^(?P[0-9]+)/activity_stream/$', InventoryActivityStreamList.as_view(), name='inventory_activity_stream_list'), re_path(r'^(?P[0-9]+)/job_templates/$', InventoryJobTemplateList.as_view(), name='inventory_job_template_list'), diff --git a/awx/api/views/inventory.py b/awx/api/views/inventory.py index 31b9cf23aeaa..7490fba79601 100644 --- a/awx/api/views/inventory.py +++ b/awx/api/views/inventory.py @@ -97,6 +97,13 @@ def destroy(self, request, *args, **kwargs): return Response(dict(error=_("{0}".format(e))), status=status.HTTP_400_BAD_REQUEST) +class InventorySourceInventoriesList(SubListAttachDetachAPIView): + model = Inventory + serializer_class = InventorySerializer + parent_model = Inventory + relationship = 'source_inventories' + + class InventoryActivityStreamList(SubListAPIView): model = ActivityStream diff --git a/awx/main/migrations/0175_constructed_inventory.py b/awx/main/migrations/0175_constructed_inventory.py new file mode 100644 index 000000000000..1c43afc9f421 --- /dev/null +++ b/awx/main/migrations/0175_constructed_inventory.py @@ -0,0 +1,82 @@ +# Generated by Django 3.2.16 on 2022-12-07 14:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0174_ensure_org_ee_admin_roles'), + ] + + operations = [ + migrations.AddField( + model_name='inventory', + name='source_inventories', + field=models.ManyToManyField( + blank=True, + help_text='Only valid for constructed inventories, this links to the inventories that will be used.', + related_name='destination_inventories', + to='main.Inventory', + ), + ), + migrations.AlterField( + model_name='inventory', + name='kind', + field=models.CharField( + blank=True, + choices=[ + ('', 'Hosts have a direct link to this inventory.'), + ('smart', 'Hosts for inventory generated using the host_filter property.'), + ('constructed', 'Parse list of source inventories with the constructed inventory plugin.'), + ], + default='', + help_text='Kind of inventory being represented.', + max_length=32, + ), + ), + migrations.AlterField( + model_name='inventorysource', + name='source', + field=models.CharField( + choices=[ + ('file', 'File, Directory or Script'), + ('constructed', 'Template additional groups and hostvars at runtime'), + ('scm', 'Sourced from a Project'), + ('ec2', 'Amazon EC2'), + ('gce', 'Google Compute Engine'), + ('azure_rm', 'Microsoft Azure Resource Manager'), + ('vmware', 'VMware vCenter'), + ('satellite6', 'Red Hat Satellite 6'), + ('openstack', 'OpenStack'), + ('rhv', 'Red Hat Virtualization'), + ('controller', 'Red Hat Ansible Automation Platform'), + ('insights', 'Red Hat Insights'), + ], + default=None, + max_length=32, + ), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source', + field=models.CharField( + choices=[ + ('file', 'File, Directory or Script'), + ('constructed', 'Template additional groups and hostvars at runtime'), + ('scm', 'Sourced from a Project'), + ('ec2', 'Amazon EC2'), + ('gce', 'Google Compute Engine'), + ('azure_rm', 'Microsoft Azure Resource Manager'), + ('vmware', 'VMware vCenter'), + ('satellite6', 'Red Hat Satellite 6'), + ('openstack', 'OpenStack'), + ('rhv', 'Red Hat Virtualization'), + ('controller', 'Red Hat Ansible Automation Platform'), + ('insights', 'Red Hat Insights'), + ], + default=None, + max_length=32, + ), + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 81af2379a072..d6ff1be5a2da 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -67,6 +67,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin): KIND_CHOICES = [ ('', _('Hosts have a direct link to this inventory.')), ('smart', _('Hosts for inventory generated using the host_filter property.')), + ('constructed', _('Parse list of source inventories with the constructed inventory plugin.')), ] class Meta: @@ -139,6 +140,12 @@ class Meta: default=None, help_text=_('Filter that will be applied to the hosts of this inventory.'), ) + source_inventories = models.ManyToManyField( + 'Inventory', + blank=True, + related_name='destination_inventories', + help_text=_('Only valid for constructed inventories, this links to the inventories that will be used.'), + ) instance_groups = OrderedManyToManyField( 'InstanceGroup', blank=True, @@ -431,12 +438,22 @@ def on_commit(): connection.on_commit(on_commit) + def _enforce_constructed_source(self): + """ + Constructed inventory should always have exactly 1 inventory source, constructed type + this enforces that requirement + """ + if self.kind == 'constructed': + if not self.inventory_sources.exists(): + self.inventory_sources.create(source='constructed', name=f'Auto-created source for: {self.name}'[:512], overwrite=True) + def save(self, *args, **kwargs): self._update_host_smart_inventory_memeberships() super(Inventory, self).save(*args, **kwargs) if self.kind == 'smart' and 'host_filter' in kwargs.get('update_fields', ['host_filter']) and connection.vendor != 'sqlite': # Minimal update of host_count for smart inventory host filter changes self.update_computed_fields() + self._enforce_constructed_source() def delete(self, *args, **kwargs): self._update_host_smart_inventory_memeberships() @@ -834,6 +851,7 @@ class InventorySourceOptions(BaseModel): SOURCE_CHOICES = [ ('file', _('File, Directory or Script')), + ('constructed', _('Template additional groups and hostvars at runtime')), ('scm', _('Sourced from a Project')), ('ec2', _('Amazon EC2')), ('gce', _('Google Compute Engine')), @@ -1364,6 +1382,8 @@ def build_env(self, inventory_update, env, private_data_dir, private_data_files) env.update(injector_env) # Preserves current behavior for Ansible change in default planned for 2.10 env['ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS'] = 'never' + # All CLOUD_PROVIDERS sources implement as inventory plugin from collection + env['ANSIBLE_INVENTORY_ENABLED'] = 'auto' return env def _get_shared_env(self, inventory_update, private_data_dir, private_data_files): @@ -1547,5 +1567,17 @@ class insights(PluginFileInjector): use_fqcn = True +class constructed(PluginFileInjector): + plugin_name = 'constructed' + namespace = 'ansible' + collection = 'builtin' + + def build_env(self, *args, **kwargs): + env = super().build_env(*args, **kwargs) + # Enable all types of inventory plugins so we pick up the script files from source inventories + del env['ANSIBLE_INVENTORY_ENABLED'] + return env + + for cls in PluginFileInjector.__subclasses__(): InventorySourceOptions.injectors[cls.__name__] = cls diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index a726a418c01e..9619cee0f1e6 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -315,17 +315,22 @@ def build_env(self, instance, private_data_dir, private_data_files=None): return env + def write_inventory_file(self, inventory, private_data_dir, file_name, script_params): + script_data = inventory.get_script_data(**script_params) + for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items(): + # maintain a list of host_name --> host_id + # so we can associate emitted events to Host objects + self.runner_callback.host_map[hostname] = hv.pop('remote_tower_id', '') + file_content = '#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json.dumps(script_data) + return self.write_private_data_file(private_data_dir, file_name, file_content, sub_dir='inventory', file_permissions=0o700) + def build_inventory(self, instance, private_data_dir): script_params = dict(hostvars=True, towervars=True) if hasattr(instance, 'job_slice_number'): script_params['slice_number'] = instance.job_slice_number script_params['slice_count'] = instance.job_slice_count - script_data = instance.inventory.get_script_data(**script_params) - # maintain a list of host_name --> host_id - # so we can associate emitted events to Host objects - self.runner_callback.host_map = {hostname: hv.pop('remote_tower_id', '') for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items()} - file_content = '#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json.dumps(script_data) - return self.write_private_data_file(private_data_dir, 'hosts', file_content, sub_dir='inventory', file_permissions=0o700) + + return self.write_inventory_file(instance.inventory, private_data_dir, 'hosts', script_params) def build_args(self, instance, private_data_dir, passwords): raise NotImplementedError @@ -1466,8 +1471,6 @@ def build_env(self, inventory_update, private_data_dir, private_data_files=None) if injector is not None: env = injector.build_env(inventory_update, env, private_data_dir, private_data_files) - # All CLOUD_PROVIDERS sources implement as inventory plugin from collection - env['ANSIBLE_INVENTORY_ENABLED'] = 'auto' if inventory_update.source == 'scm': for env_k in inventory_update.source_vars_dict: @@ -1520,6 +1523,15 @@ def build_args(self, inventory_update, private_data_dir, passwords): args = ['ansible-inventory', '--list', '--export'] + # special case for constructed inventories, we pass source inventories from database + # these must come in order, and in order _before_ the constructed inventory itself + if inventory_update.inventory.kind == 'constructed': + for source_inventory in inventory_update.inventory.source_inventories.all(): + args.append('-i') + script_params = dict(hostvars=True, towervars=True) + source_inv_path = self.write_inventory_file(source_inventory, private_data_dir, f'hosts_{source_inventory.id}', script_params) + args.append(to_container_path(source_inv_path, private_data_dir)) + # Add arguments for the source inventory file/script/thing rel_path = self.pseudo_build_inventory(inventory_update, private_data_dir) container_location = os.path.join(CONTAINER_ROOT, rel_path) diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index d246853c836d..36a7af04ce13 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -169,7 +169,8 @@ def test_all_cloud_sources_covered(self): CLOUD_PROVIDERS constant contains the same names as what are defined within the injectors """ - assert set(CLOUD_PROVIDERS) == set(InventorySource.injectors.keys()) + # slight exception case for constructed, because it has a FQCN but is not a cloud source + assert set(CLOUD_PROVIDERS) | set(['constructed']) == set(InventorySource.injectors.keys()) @pytest.mark.parametrize('source,filename', [('ec2', 'aws_ec2.yml'), ('openstack', 'openstack.yml'), ('gce', 'gcp_compute.yml')]) def test_plugin_filenames(self, source, filename): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 5488f5041228..393247cf1b2f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -745,6 +745,11 @@ SCM_EXCLUDE_EMPTY_GROUPS = False # SCM_INSTANCE_ID_VAR = +# ---------------- +# -- Constructed -- +# ---------------- +CONSTRUCTED_EXCLUDE_EMPTY_GROUPS = False + # --------------------- # -- Activity Stream -- # ---------------------