|
| 1 | +""" |
| 2 | +Python Module to interact with aws opensearch instance |
| 3 | +""" |
| 4 | + |
| 5 | +import os |
| 6 | +import string |
| 7 | +import random |
| 8 | +from datetime import datetime |
| 9 | + |
| 10 | +import yaml |
| 11 | +import boto3 |
| 12 | +from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth |
| 13 | + |
| 14 | +# Create an index with non-default settings. |
| 15 | +def create_index(os_client, index_name, number_of_shards=4): |
| 16 | + """ Method to create index in opensearch instance """ |
| 17 | + index_body = { |
| 18 | + 'settings': { |
| 19 | + 'index': { |
| 20 | + 'number_of_shards': number_of_shards |
| 21 | + } |
| 22 | + } |
| 23 | + } |
| 24 | + response = os_client.indices.create(index_name, body=index_body) |
| 25 | + print('\nCreating index:') |
| 26 | + print(response) |
| 27 | + |
| 28 | +def delete_index(os_client, index_name): |
| 29 | + """ Method to delete index from opensearch instance """ |
| 30 | + response = os_client.indices.delete( |
| 31 | + index = index_name |
| 32 | + ) |
| 33 | + print('\nDeleting index:') |
| 34 | + print(response) |
| 35 | + |
| 36 | +def add_document_to_index(os_client, index_name, doc_id, document): |
| 37 | + """ Add a document to the index """ |
| 38 | + response = os_client.index( |
| 39 | + index = index_name, |
| 40 | + body = document, |
| 41 | + id = doc_id, |
| 42 | + refresh = True |
| 43 | + ) |
| 44 | + print('\nAdding document:') |
| 45 | + print(response) |
| 46 | + |
| 47 | +def delete_a_document(os_client, index_name, doc_id): |
| 48 | + """ Delete a document from index """ |
| 49 | + response = os_client.delete( |
| 50 | + index = index_name, |
| 51 | + id = doc_id |
| 52 | + ) |
| 53 | + print('\nDeleting document:') |
| 54 | + print(response) |
| 55 | + |
| 56 | +def search_document(os_client, index_name): |
| 57 | + """ Sample search for the document """ |
| 58 | + qval = 'miller' |
| 59 | + query = { |
| 60 | + 'size': 5, |
| 61 | + 'query': { |
| 62 | + 'multi_match': { |
| 63 | + 'query': qval, |
| 64 | + 'fields': ['title^2', 'director'] |
| 65 | + } |
| 66 | + } |
| 67 | + } |
| 68 | + response = os_client.search( |
| 69 | + body = query, |
| 70 | + index = index_name |
| 71 | + ) |
| 72 | + print('\nSearch results:') |
| 73 | + print(response) |
| 74 | + |
| 75 | +def setup_os_client(): |
| 76 | + """ Setup the open search client """ |
| 77 | + host = os.environ['OS_HOST'] #cluster endpoint, for ex: my-domain.us-east-1.es.amazonaws.com |
| 78 | + region = os.environ['OS_REGION'] |
| 79 | + credentials = boto3.Session().get_credentials() |
| 80 | + auth = AWSV4SignerAuth(credentials, region) |
| 81 | + |
| 82 | + os_client = OpenSearch( |
| 83 | + hosts = [{'host': host, 'port': 443}], |
| 84 | + http_auth = auth, |
| 85 | + use_ssl = True, |
| 86 | + verify_certs = True, |
| 87 | + connection_class = RequestsHttpConnection |
| 88 | + ) |
| 89 | + return os_client |
| 90 | + |
| 91 | +def read_metric_data(file_name): |
| 92 | + """ Read from the metrics file and construct the document """ |
| 93 | + with open(file_name, encoding="utf-8") as file_d: |
| 94 | + content = yaml.load(file_d, Loader=yaml.FullLoader) |
| 95 | + kpi_data = content['kpi'] |
| 96 | + for data in kpi_data: |
| 97 | + if data['name'] == 'usage': |
| 98 | + for metric in data['metrics']: |
| 99 | + if metric['name'] == 'Memory_MiB': |
| 100 | + memory_average = metric['average'] |
| 101 | + memory_maximum = metric['maximum'] |
| 102 | + elif metric['name'] == 'CPU_millicores': |
| 103 | + cpu_average = metric['average'] |
| 104 | + cpu_maximum = metric['maximum'] |
| 105 | + dt_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| 106 | + return {'upload_date': dt_string, |
| 107 | + 'memory_average' : float(memory_average), |
| 108 | + 'memory_maximum': float(memory_maximum), |
| 109 | + 'cpu_average': float(cpu_average), |
| 110 | + 'cpu_maximum': float(cpu_maximum), |
| 111 | + 'memory_average_threshold': get_average_threshold_mem(), |
| 112 | + 'memory_maximum_threshold': get_maximum_threshold_mem(), |
| 113 | + 'cpu_average_threshold': get_average_threshold_cpu(), |
| 114 | + 'cpu_maximum_threshold': get_maximum_threshold_cpu()} |
| 115 | + |
| 116 | +def generate_id(): |
| 117 | + """ Generate a random id of length 6 """ |
| 118 | + length = 6 |
| 119 | + return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) |
| 120 | + |
| 121 | +def get_average_threshold_mem(): |
| 122 | + """ returns current threshold value 150 """ |
| 123 | + return float(os.environ['TEST_PERFORMANCE_AVG_MEMORY']) |
| 124 | + |
| 125 | +def get_maximum_threshold_mem(): |
| 126 | + """ returns current threshold value 200 """ |
| 127 | + return float(os.environ['TEST_PERFORMANCE_MAX_MEMORY']) |
| 128 | + |
| 129 | +def get_average_threshold_cpu(): |
| 130 | + """ returns current thresholds value 20 """ |
| 131 | + return float(os.environ['TEST_PERFORMANCE_AVG_CPU']) |
| 132 | + |
| 133 | +def get_maximum_threshold_cpu(): |
| 134 | + """ returns current thresholds value 100 """ |
| 135 | + return float(os.environ['TEST_PERFORMANCE_MAX_CPU']) |
| 136 | + |
| 137 | +if __name__ == '__main__': |
| 138 | + OS_INDEX_NAME = 'sbo-perf-data' |
| 139 | + client = setup_os_client() |
| 140 | + #create_index(client, index_name) |
| 141 | + #delete_index(client, index_name) |
| 142 | + # delete_a_document(client, index_name, id) |
| 143 | + |
| 144 | + metric_file_name = os.environ['KPI_YAML_FILE'] |
| 145 | + doc = read_metric_data(metric_file_name) |
| 146 | + #doc = {'upload_date': '2022-12-14 06:30:30', |
| 147 | + # 'memory_average' : 68.2, |
| 148 | + # 'memory_maximum': 98.2, |
| 149 | + # 'cpu_average': 10.5, |
| 150 | + # 'cpu_maximum': 90.2, |
| 151 | + # 'memory_average_threshold': 150, |
| 152 | + # 'memory_maximum_threshold': 200, |
| 153 | + # 'cpu_average_threshold': 20, |
| 154 | + # 'cpu_maximum_threshold': 100} |
| 155 | + RANDOM_DOC_ID = generate_id() |
| 156 | + print(f"Random Generated ID: {RANDOM_DOC_ID}") |
| 157 | + add_document_to_index(client, OS_INDEX_NAME, RANDOM_DOC_ID, doc) |
0 commit comments