Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added new env to run multiple policies #889

Merged
merged 1 commit into from
Jan 16, 2025
Merged

Added new env to run multiple policies #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cloud_governance/main/environment_variables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ def __init__(self):

self._environment_variables_dict['DAYS_TO_TAKE_ACTION'] = int(
EnvironmentVariables.get_env('DAYS_TO_TAKE_ACTION', "7"))
if not hasattr(self, 'POLICIES_LIST'):
self.POLICIES_LIST = EnvironmentVariables.get_env('POLICIES_LIST')

self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS',
True)
Expand Down
266 changes: 141 additions & 125 deletions cloud_governance/main/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,22 @@
from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp, logger
from cloud_governance.policy.policy_operations.aws.tag_cluster.run_tag_cluster_resouces import tag_cluster_resource, \
remove_cluster_resources_tags
from cloud_governance.policy.policy_operations.aws.tag_non_cluster.run_tag_non_cluster_resources import tag_non_cluster_resource, \
from cloud_governance.policy.policy_operations.aws.tag_non_cluster.run_tag_non_cluster_resources import \
tag_non_cluster_resource, \
remove_tag_non_cluster_resource, tag_na_resources
from cloud_governance.policy.policy_operations.aws.tag_user.run_tag_iam_user import tag_iam_user, run_validate_iam_user_tags
from cloud_governance.policy.policy_operations.aws.zombie_cluster.run_zombie_cluster_resources import zombie_cluster_resource
from cloud_governance.policy.policy_operations.aws.tag_user.run_tag_iam_user import tag_iam_user, \
run_validate_iam_user_tags
from cloud_governance.policy.policy_operations.aws.zombie_cluster.run_zombie_cluster_resources import \
zombie_cluster_resource
from cloud_governance.policy.policy_operations.gcp.gcp_policy_runner import GcpPolicyRunner
from cloud_governance.policy.policy_operations.gitleaks.gitleaks import GitLeaks
from cloud_governance.policy.policy_operations.ibm.ibm_operations.ibm_policy_runner import IBMPolicyRunner
from cloud_governance.main.environment_variables import environment_variables
from cloud_governance.main.es_uploader import ESUploader
from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
from cloud_governance.policy.policy_operations.aws.zombie_cluster.validate_zombies import ValidateZombies
from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.zombie_non_cluster_polices import ZombieNonClusterPolicies

from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.zombie_non_cluster_polices import \
ZombieNonClusterPolicies

environment_variables_dict = environment_variables.environment_variables_dict
log_level = environment_variables_dict.get('log_level', 'INFO').upper()
Expand Down Expand Up @@ -197,138 +200,151 @@ def main():
:return: the action output
"""
# environment variables - get while running the docker
region_env = environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
dry_run = environment_variables_dict.get('dry_run', 'yes')
policies_list = environment_variables.POLICIES_LIST
if not policies_list:
policies_list = [environment_variables_dict.get('policy').strip()]
else:
if isinstance(policies_list, str):
policies_list = policies_list.split(',')
logger.info(f"Running polices: {policies_list}")
for policy in policies_list:
environment_variables_dict['policy'] = policy.strip()
region_env = environment_variables_dict.get('AWS_DEFAULT_REGION', 'us-east-2')
dry_run = environment_variables_dict.get('dry_run', 'yes')

account = environment_variables_dict.get('account', '')
policy = environment_variables_dict.get('policy', '')
upload_data_es = environment_variables_dict.get('upload_data_es', '')
es_host = environment_variables_dict.get('es_host', '')
es_port = environment_variables_dict.get('es_port', '')
es_index = environment_variables_dict.get('es_index', '')
es_doc_type = environment_variables_dict.get('es_doc_type', '')
bucket = environment_variables_dict.get('bucket', '')
main_operations = MainOperations()
response = main_operations.run()
if not response:
if environment_variables_dict.get('COMMON_POLICIES'):
run_common_policies()
elif environment_variables_dict.get('CLOUD_RESOURCE_ORCHESTRATION'):
run_cloud_resource_orchestration()
else:
non_cluster_polices_runner = None
is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies')
if is_non_cluster_polices_runner:
non_cluster_polices_runner = ZombieNonClusterPolicies()
account = environment_variables_dict.get('account', '')
policy = environment_variables_dict.get('policy', '')
upload_data_es = environment_variables_dict.get('upload_data_es', '')
es_host = environment_variables_dict.get('es_host', '')
es_port = environment_variables_dict.get('es_port', '')
es_index = environment_variables_dict.get('es_index', '')
es_doc_type = environment_variables_dict.get('es_doc_type', '')
bucket = environment_variables_dict.get('bucket', '')
main_operations = MainOperations()
response = main_operations.run()
if not response:
if environment_variables_dict.get('COMMON_POLICIES'):
run_common_policies()
elif environment_variables_dict.get('CLOUD_RESOURCE_ORCHESTRATION'):
run_cloud_resource_orchestration()
else:
non_cluster_polices_runner = None
is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies')
if is_non_cluster_polices_runner:
non_cluster_polices_runner = ZombieNonClusterPolicies()

ibm_classic_infrastructure_policy_runner = None
is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies')
if not is_tag_ibm_classic_infrastructure_runner:
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'IBM':
is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('cost_policies')
if is_tag_ibm_classic_infrastructure_runner:
ibm_classic_infrastructure_policy_runner = IBMPolicyRunner()
ibm_classic_infrastructure_policy_runner = None
is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies')
if not is_tag_ibm_classic_infrastructure_runner:
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get(
'PUBLIC_CLOUD_NAME').upper() == 'IBM':
is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get(
'cost_policies')
if is_tag_ibm_classic_infrastructure_runner:
ibm_classic_infrastructure_policy_runner = IBMPolicyRunner()

is_cost_explorer_policies_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AWS':
cost_explorer_policies_runner = None
is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies')
if is_cost_explorer_policies_runner:
cost_explorer_policies_runner = CostReportPolicies()
is_cost_explorer_policies_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AWS':
cost_explorer_policies_runner = None
is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies')
if is_cost_explorer_policies_runner:
cost_explorer_policies_runner = CostReportPolicies()

is_azure_policy_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AZURE':
azure_cost_policy_runner = None
is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies')
if is_azure_policy_runner:
azure_cost_policy_runner = AzurePolicyRunner()
is_azure_policy_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get(
'PUBLIC_CLOUD_NAME').upper() == 'AZURE':
azure_cost_policy_runner = None
is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies')
if is_azure_policy_runner:
azure_cost_policy_runner = AzurePolicyRunner()

is_gcp_policy_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'GCP':
gcp_cost_policy_runner = None
is_gcp_policy_runner = policy in environment_variables_dict.get('cost_policies')
if is_gcp_policy_runner:
gcp_cost_policy_runner = GcpPolicyRunner()
is_gcp_policy_runner = ''
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get(
'PUBLIC_CLOUD_NAME').upper() == 'GCP':
gcp_cost_policy_runner = None
is_gcp_policy_runner = policy in environment_variables_dict.get('cost_policies')
if is_gcp_policy_runner:
gcp_cost_policy_runner = GcpPolicyRunner()

@logger_time_stamp
def run_non_cluster_polices_runner():
"""
This method run the aws non-cluster policies
@return:
"""
non_cluster_polices_runner.run()
@logger_time_stamp
def run_non_cluster_polices_runner():
"""
This method run the aws non-cluster policies
@return:
"""
non_cluster_polices_runner.run()

def run_tag_ibm_classic_infrastructure_runner():
"""
This method run the IBM policies
@return:
"""
ibm_classic_infrastructure_policy_runner.run()
def run_tag_ibm_classic_infrastructure_runner():
"""
This method run the IBM policies
@return:
"""
ibm_classic_infrastructure_policy_runner.run()

@logger_time_stamp
def run_cost_explorer_policies_runner():
"""
This method run the aws cost_explorer policies
@return:
"""
cost_explorer_policies_runner.run()
@logger_time_stamp
def run_cost_explorer_policies_runner():
"""
This method run the aws cost_explorer policies
@return:
"""
cost_explorer_policies_runner.run()

@logger_time_stamp
def run_azure_policy_runner():
"""
This method run the azure policies
@return:
"""
azure_cost_policy_runner.run()
@logger_time_stamp
def run_azure_policy_runner():
"""
This method run the azure policies
@return:
"""
azure_cost_policy_runner.run()

@logger_time_stamp
def run_gcp_policy_runner():
"""
This method run the gcp policies
"""
gcp_cost_policy_runner.run()
@logger_time_stamp
def run_gcp_policy_runner():
"""
This method run the gcp policies
"""
gcp_cost_policy_runner.run()

# 1. ELK Uploader
if upload_data_es:
input_data = {'es_host': es_host,
'es_port': int(es_port),
'es_index': es_index,
'es_doc_type': es_doc_type,
'es_add_items': {'account': account},
'bucket': bucket,
'logs_bucket_key': 'logs',
's3_file_name': 'resources.json',
'region': region_env,
'policy': policy,
}
elk_uploader = ESUploader(**input_data)
elk_uploader.upload_to_es(account=account)
# 2. POLICY
elif is_non_cluster_polices_runner:
run_non_cluster_polices_runner()
elif is_tag_ibm_classic_infrastructure_runner:
run_tag_ibm_classic_infrastructure_runner()
elif is_cost_explorer_policies_runner:
run_cost_explorer_policies_runner()
elif is_azure_policy_runner:
run_azure_policy_runner()
elif is_gcp_policy_runner:
run_gcp_policy_runner()
else:
if not policy:
logger.exception(f'Missing Policy name: "{policy}"')
raise Exception(f'Missing Policy name: "{policy}"')
if region_env == 'all':
# must be set for boto3 client default region
# environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
ec2 = boto3.client('ec2')
regions_data = ec2.describe_regions()
for region in regions_data['Regions']:
# logger.info(f"region: {region['RegionName']}")
environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName']
run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run)
# 1. ELK Uploader
if upload_data_es:
input_data = {'es_host': es_host,
'es_port': int(es_port),
'es_index': es_index,
'es_doc_type': es_doc_type,
'es_add_items': {'account': account},
'bucket': bucket,
'logs_bucket_key': 'logs',
's3_file_name': 'resources.json',
'region': region_env,
'policy': policy,
}
elk_uploader = ESUploader(**input_data)
elk_uploader.upload_to_es(account=account)
# 2. POLICY
elif is_non_cluster_polices_runner:
run_non_cluster_polices_runner()
elif is_tag_ibm_classic_infrastructure_runner:
run_tag_ibm_classic_infrastructure_runner()
elif is_cost_explorer_policies_runner:
run_cost_explorer_policies_runner()
elif is_azure_policy_runner:
run_azure_policy_runner()
elif is_gcp_policy_runner:
run_gcp_policy_runner()
else:
run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run)
if not policy:
logger.exception(f'Missing Policy name: "{policy}"')
raise Exception(f'Missing Policy name: "{policy}"')
if region_env == 'all':
# must be set for boto3 client default region
# environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
ec2 = boto3.client('ec2')
regions_data = ec2.describe_regions()
for region in regions_data['Regions']:
# logger.info(f"region: {region['RegionName']}")
environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName']
run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run)
else:
run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run)


if __name__ == '__main__':
Expand Down
Loading