|
| 1 | +""" |
| 2 | +Python Module to interact with aws opensearch instance |
| 3 | +""" |
| 4 | + |
| 5 | +import os |
| 6 | +import string |
| 7 | +import random |
| 8 | +from datetime import datetime |
| 9 | + |
| 10 | +import yaml |
| 11 | +import boto3 |
| 12 | +from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth |
| 13 | + |
| 14 | + |
| 15 | +def create_index(os_client, index_name, number_of_shards=4): |
| 16 | + """ Method to create index in opensearch instance """ |
| 17 | + index_body = { |
| 18 | + 'settings': { |
| 19 | + 'index': { |
| 20 | + 'number_of_shards': number_of_shards |
| 21 | + } |
| 22 | + } |
| 23 | + } |
| 24 | + response = os_client.indices.create(index_name, body=index_body) |
| 25 | + print('\nCreating index:') |
| 26 | + print(response) |
| 27 | + |
| 28 | + |
| 29 | +def delete_index(os_client, index_name): |
| 30 | + """ Method to delete index from opensearch instance """ |
| 31 | + response = os_client.indices.delete( |
| 32 | + index=index_name |
| 33 | + ) |
| 34 | + print('\nDeleting index:') |
| 35 | + print(response) |
| 36 | + |
| 37 | + |
| 38 | +def add_document_to_index(os_client, index_name, doc_id, document): |
| 39 | + """ Add a document to the index """ |
| 40 | + response = os_client.index( |
| 41 | + index=index_name, |
| 42 | + body=document, |
| 43 | + id=doc_id, |
| 44 | + refresh=True |
| 45 | + ) |
| 46 | + print('\nAdding document:') |
| 47 | + print(response) |
| 48 | + |
| 49 | + |
| 50 | +def delete_a_document(os_client, index_name, doc_id): |
| 51 | + """ Delete a document from index """ |
| 52 | + response = os_client.delete( |
| 53 | + index=index_name, |
| 54 | + id=doc_id |
| 55 | + ) |
| 56 | + print('\nDeleting document:') |
| 57 | + print(response) |
| 58 | + |
| 59 | + |
| 60 | +def search_document(os_client, index_name): |
| 61 | + """ Sample search for the document """ |
| 62 | + qval = 'miller' |
| 63 | + query = { |
| 64 | + 'size': 5, |
| 65 | + 'query': { |
| 66 | + 'multi_match': { |
| 67 | + 'query': qval, |
| 68 | + 'fields': ['title^2', 'director'] |
| 69 | + } |
| 70 | + } |
| 71 | + } |
| 72 | + response = os_client.search( |
| 73 | + body=query, |
| 74 | + index=index_name |
| 75 | + ) |
| 76 | + print('\nSearch results:') |
| 77 | + print(response) |
| 78 | + |
| 79 | + |
| 80 | +def setup_os_client(): |
| 81 | + """ Setup the open search client """ |
| 82 | + host = os.environ['OS_HOST'] # cluster endpoint, for ex: my-domain.us-east-1.es.amazonaws.com |
| 83 | + region = os.environ['OS_REGION'] |
| 84 | + credentials = boto3.Session().get_credentials() |
| 85 | + auth = AWSV4SignerAuth(credentials, region) |
| 86 | + |
| 87 | + os_client = OpenSearch( |
| 88 | + hosts=[{'host': host, 'port': 443}], |
| 89 | + http_auth=auth, |
| 90 | + use_ssl=True, |
| 91 | + verify_certs=True, |
| 92 | + connection_class=RequestsHttpConnection |
| 93 | + ) |
| 94 | + return os_client |
| 95 | + |
| 96 | + |
| 97 | +def read_metric_data(file_name): |
| 98 | + """ Read from the metrics file and construct the document """ |
| 99 | + with open(file_name, encoding="utf-8") as file_d: |
| 100 | + content = yaml.load(file_d, Loader=yaml.FullLoader) |
| 101 | + |
| 102 | + kpi_data = content['kpi'] |
| 103 | + execution_timestamp = content['execution_timestamp'] |
| 104 | + sbo_version = content['sbo_version'] |
| 105 | + openshift_version = content['openshift_version'] |
| 106 | + openshift_release = content['openshift_release'] |
| 107 | + run_type = content['run_type'] |
| 108 | + pull_number = content['pull_number'] |
| 109 | + commit_id = content['commit_id'] |
| 110 | + |
| 111 | + for data in kpi_data: |
| 112 | + if data['name'] == 'usage': |
| 113 | + for metric in data['metrics']: |
| 114 | + if metric['name'] == 'Memory_MiB': |
| 115 | + memory_average = metric['average'] |
| 116 | + memory_maximum = metric['maximum'] |
| 117 | + elif metric['name'] == 'CPU_millicores': |
| 118 | + cpu_average = metric['average'] |
| 119 | + cpu_maximum = metric['maximum'] |
| 120 | + dt_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| 121 | + return {'upload_date': dt_string, |
| 122 | + 'memory_average': float(memory_average), |
| 123 | + 'memory_maximum': float(memory_maximum), |
| 124 | + 'cpu_average': float(cpu_average), |
| 125 | + 'cpu_maximum': float(cpu_maximum), |
| 126 | + 'memory_average_threshold': get_average_threshold_mem(), |
| 127 | + 'memory_maximum_threshold': get_maximum_threshold_mem(), |
| 128 | + 'cpu_average_threshold': get_average_threshold_cpu(), |
| 129 | + 'cpu_maximum_threshold': get_maximum_threshold_cpu(), |
| 130 | + 'run_type': run_type, |
| 131 | + 'pull_number': pull_number, |
| 132 | + 'commit_id': commit_id, |
| 133 | + 'execution_timestamp': execution_timestamp, |
| 134 | + 'sbo_version': sbo_version, |
| 135 | + 'openshift_version': openshift_version, |
| 136 | + 'openshift_release': openshift_release} |
| 137 | + |
| 138 | + |
| 139 | +def generate_id(): |
| 140 | + """ Generate a random id of length 6 """ |
| 141 | + length = 6 |
| 142 | + return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) |
| 143 | + |
| 144 | + |
| 145 | +def get_average_threshold_mem(): |
| 146 | + """ returns current threshold value 150 """ |
| 147 | + return float(os.environ['TEST_PERFORMANCE_AVG_MEMORY']) |
| 148 | + |
| 149 | + |
| 150 | +def get_maximum_threshold_mem(): |
| 151 | + """ returns current threshold value 200 """ |
| 152 | + return float(os.environ['TEST_PERFORMANCE_MAX_MEMORY']) |
| 153 | + |
| 154 | + |
| 155 | +def get_average_threshold_cpu(): |
| 156 | + """ returns current thresholds value 20 """ |
| 157 | + return float(os.environ['TEST_PERFORMANCE_AVG_CPU']) |
| 158 | + |
| 159 | + |
| 160 | +def get_maximum_threshold_cpu(): |
| 161 | + """ returns current thresholds value 100 """ |
| 162 | + return float(os.environ['TEST_PERFORMANCE_MAX_CPU']) |
| 163 | + |
| 164 | + |
| 165 | +if __name__ == '__main__': |
| 166 | + OS_INDEX_NAME = 'sbo-perf-data' |
| 167 | + client = setup_os_client() |
| 168 | + # create_index(client, index_name) |
| 169 | + # delete_index(client, index_name) |
| 170 | + # delete_a_document(client, index_name, id) |
| 171 | + |
| 172 | + metric_file_name = os.environ['KPI_YAML_FILE'] |
| 173 | + doc = read_metric_data(metric_file_name) |
| 174 | + # doc = {'upload_date': '2022-12-14 06:30:30', |
| 175 | + # 'memory_average' : 68.2, |
| 176 | + # 'memory_maximum': 98.2, |
| 177 | + # 'cpu_average': 10.5, |
| 178 | + # 'cpu_maximum': 90.2, |
| 179 | + # 'memory_average_threshold': 150, |
| 180 | + # 'memory_maximum_threshold': 200, |
| 181 | + # 'cpu_average_threshold': 20, |
| 182 | + # 'cpu_maximum_threshold': 100} |
| 183 | + RANDOM_DOC_ID = generate_id() |
| 184 | + print(f"Random Generated ID: {RANDOM_DOC_ID}") |
| 185 | + add_document_to_index(client, OS_INDEX_NAME, RANDOM_DOC_ID, doc) |
0 commit comments