[Fetch Migration] Include index aliases in metadata migration (opense…

…arch-project#429) Small update to the metadata migration step of Fetch Migration to include index aliases during metadata migration. This PR also includes unit test updates, as well as a minor fix to the Data Prepper pipeline template file used by the CDK to remove an invalid flag (Data Prepper does not require explicit configuration to toggle TLS). --------- Signed-off-by: Kartik Ganesh <gkart@amazon.com>
kartg · Nov 15, 2023 · df78bb0 · df78bb0
1 parent 68ccfc1
commit df78bb0
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 2 deletions.
diff --git a/FetchMigration/python/index_operations.py b/FetchMigration/python/index_operations.py
@@ -8,6 +8,7 @@
 
 SETTINGS_KEY = "settings"
 MAPPINGS_KEY = "mappings"
+ALIASES_KEY = "aliases"
 COUNT_KEY = "count"
 __INDEX_KEY = "index"
 __ALL_INDICES_ENDPOINT = "*"
@@ -43,6 +44,8 @@ def create_indices(indices_data: dict, endpoint: EndpointInfo):
         data_dict = dict()
         data_dict[SETTINGS_KEY] = indices_data[index][SETTINGS_KEY]
         data_dict[MAPPINGS_KEY] = indices_data[index][MAPPINGS_KEY]
+        if ALIASES_KEY in indices_data[index] and len(indices_data[index][ALIASES_KEY]) > 0:
+            data_dict[ALIASES_KEY] = indices_data[index][ALIASES_KEY]
         try:
             resp = requests.put(index_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl(),
                                 json=data_dict)

diff --git a/FetchMigration/python/tests/test_constants.py b/FetchMigration/python/tests/test_constants.py
@@ -10,11 +10,13 @@
 INDEX3_NAME = "index3"
 SETTINGS_KEY = "settings"
 MAPPINGS_KEY = "mappings"
+ALIASES_KEY = "aliases"
 INDEX_KEY = "index"
 NUM_SHARDS_SETTING = "number_of_shards"
 NUM_REPLICAS_SETTING = "number_of_replicas"
 BASE_INDICES_DATA = {
     INDEX1_NAME: {
+        ALIASES_KEY: {},
         SETTINGS_KEY: {
             INDEX_KEY: {
                 # Internal data
@@ -31,6 +33,9 @@
         }
     },
     INDEX2_NAME: {
+        ALIASES_KEY: {
+            "alias2": {}
+        },
         SETTINGS_KEY: {
             INDEX_KEY: {
                 NUM_SHARDS_SETTING: 2,
@@ -42,6 +47,9 @@
         }
     },
     INDEX3_NAME: {
+        ALIASES_KEY: {
+            "alias3": {"filter": {"term": {"id": "test"}}}
+        },
         SETTINGS_KEY: {
             INDEX_KEY: {
                 NUM_SHARDS_SETTING: 1,

diff --git a/FetchMigration/python/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py
@@ -46,6 +46,23 @@ def test_create_indices(self):
                       match=[matchers.json_params_matcher(test_data[test_constants.INDEX3_NAME])])
         index_operations.create_indices(test_data, EndpointInfo(test_constants.TARGET_ENDPOINT))
 
+    @responses.activate
+    def test_create_indices_empty_alias(self):
+        aliases_key = "aliases"
+        test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA)
+        del test_data[test_constants.INDEX2_NAME]
+        del test_data[test_constants.INDEX3_NAME]
+        # Setup expected create payload without "aliases"
+        expected_payload = copy.deepcopy(test_data[test_constants.INDEX1_NAME])
+        del expected_payload[aliases_key]
+        responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX1_NAME,
+                      match=[matchers.json_params_matcher(expected_payload)])
+        # Empty "aliases" should be stripped
+        index_operations.create_indices(test_data, EndpointInfo(test_constants.TARGET_ENDPOINT))
+        # Index without "aliases" should not fail
+        del test_data[test_constants.INDEX1_NAME][aliases_key]
+        index_operations.create_indices(test_data, EndpointInfo(test_constants.TARGET_ENDPOINT))
+
     @responses.activate
     def test_create_indices_exception(self):
         # Set up expected PUT calls with a mock response status

diff --git a/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml b/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml
@@ -7,8 +7,6 @@ historical-data-migration:
       # unless the file is being used outside of the CDK
       hosts:
       - <SOURCE_CLUSTER_HOST>
-      # Uncomment the following line to disable TLS
-      #insecure: true
       # Example configuration on how to disable authentication (default: false)
       disable_authentication: true
       indices: