AMBARI-24024 - Logsearch: service advisor does not set recommended number of shards (#1457)

2 files changed, 100 insertions, 61 deletions

diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
index 059f59324b..2e4acca37b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/service_advisor.py
@@ -21,13 +21,6 @@ limitations under the License.
 import imp
 import os
 import traceback
-import re
-import socket
-import fnmatch
-import math
-
-
-from resource_management.core.logger import Logger
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -140,34 +133,25 @@ class LogSearchServiceAdvisor(service_advisor.ServiceAdvisor):
       infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA_SOLR", "INFRA_SOLR")
       # if there is AMBARI_INFRA, calculate the min/max shards and recommendations based on the number of infra solr hosts
       if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
-        replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
-        recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
-        
         recommendedMinShards = len(infraSolrHosts)
         recommendedShards = 2 * len(infraSolrHosts)
-        recommendedMaxShards = max(3 * len(infraSolrHosts), 5)
-      # if there is no AMBARI_INFRA (i.e. external solr is used), use default values for min/max shards and recommendations
+        recommendedMaxShards = 10 * len(infraSolrHosts)
       else:
-        recommendedReplicationFactor = 2
-        
+        # if there is no AMBARI_INFRA (i.e. external solr is used), use default values for min/max shards and recommendations
         recommendedMinShards = 1
         recommendedShards = 1
         recommendedMaxShards = 100
-        
         putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true')
         
-        # recommend number of shard
-        putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
-        putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
-        putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
-      
-        putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
-        putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
-        putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
-        # recommend replication factor
-        putLogSearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
-        putLogSearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
-      
+      # recommend number of shard
+      putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+      putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+      putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
+
+      putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+      putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+      putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+
     kerberos_authentication_enabled = self.isSecurityEnabled(services)
     # if there is no kerberos enabled hide kerberor related properties
     if not kerberos_authentication_enabled:
diff --git a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
index 37b02f4813..c3a79a0ee0 100644
--- a/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
+++ b/ambari-server/src/test/python/common-services/LOGSEARCH/test_service_advisor.py
@@ -17,12 +17,9 @@ limitations under the License.
 """
 
 import imp
-import json
 import os
 from unittest import TestCase
 
-from mock.mock import patch, MagicMock
-
 
 class TestLOGSEARCH050ServiceAdvisor(TestCase):
 
@@ -41,27 +38,46 @@ class TestLOGSEARCH050ServiceAdvisor(TestCase):
   with open(logserch050ServiceAdvisorPath, 'rb') as fp:
     service_advisor_impl = imp.load_module('service_advisor_impl', fp, logserch050ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
 
+  configurations = {
+    "logsearch-properties": {
+    }
+  }
+
+  clusterData = {
+    "cpu": 4,
+    "mapMemory": 3000,
+    "amMemory": 2000,
+    "reduceMemory": 2056,
+    "containers": 3,
+    "ramPerContainer": 256
+  }
+
+  hosts = {
+    "items" : [
+      {
+        "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+        "Hosts" : {
+          "cpu_count" : 1,
+          "host_name" : "c6401.ambari.apache.org",
+          "os_arch" : "x86_64",
+          "os_type" : "centos6",
+          "ph_cpu_count" : 1,
+          "public_host_name" : "c6401.ambari.apache.org",
+          "rack_info" : "/default-rack",
+          "total_mem" : 1922680
+        }
+      }
+    ]
+  }
+
   def setUp(self):
     serviceAdvisorClass = getattr(self.service_advisor_impl, 'LogSearchServiceAdvisor')
     self.serviceAdvisor = serviceAdvisorClass()
 
   def test_recommendLogsearchConfiguration(self):
-    configurations = {
-      "logsearch-properties": {
-      }
-    }
-
-    clusterData = {
-      "cpu": 4,
-      "mapMemory": 3000,
-      "amMemory": 2000,
-      "reduceMemory": 2056,
-      "containers": 3,
-      "ramPerContainer": 256
-    }
     expected = {
       'logsearch-properties': {
-        'properties': {}
+        'properties': {'logsearch.collection.service.logs.numshards': '4', 'logsearch.collection.audit.logs.numshards': '4'}, 'property_attributes': {'logsearch.collection.service.logs.numshards': {'minimum': '2', 'maximum': '20'}, 'logsearch.collection.audit.logs.numshards': {'minimum': '2', 'maximum': '20'}}
       },
       'logfeeder-env': {
         'property_attributes': {
@@ -123,25 +139,64 @@ class TestLOGSEARCH050ServiceAdvisor(TestCase):
 
     }
 
-    hosts = {
-      "items" : [
-        {
-          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
-          "Hosts" : {
-            "cpu_count" : 1,
-            "host_name" : "c6401.ambari.apache.org",
-            "os_arch" : "x86_64",
-            "os_type" : "centos6",
-            "ph_cpu_count" : 1,
-            "public_host_name" : "c6401.ambari.apache.org",
-            "rack_info" : "/default-rack",
-            "total_mem" : 1922680
+    def return_c6401_hostname(services, service_name, component_name):
+      return ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+    self.serviceAdvisor.getComponentHostNames = return_c6401_hostname
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.configurations, self.clusterData, services, self.hosts)
+    self.assertEquals(self.configurations, expected)
+
+  def test_recommendLogsearchConfigurationWhenSolrIsExternal(self):
+    expected = {
+      'logsearch-properties': {
+        'properties': {
+          'logsearch.collection.service.logs.numshards': '1',
+          'logsearch.collection.audit.logs.numshards': '1'
+        },
+        'property_attributes': {
+          'logsearch.collection.service.logs.numshards': {'minimum': '1', 'maximum': '100'},
+          'logsearch.collection.audit.logs.numshards': {'minimum': '1', 'maximum': '100'}
+        }
+      },
+      'logsearch-env': {
+        'property_attributes': {
+          'logsearch_external_solr_kerberos_principal': {'visible': 'false'},
+          'logsearch_external_solr_kerberos_keytab': {'visible': 'false'}
+        }
+      },
+      'logfeeder-env': {
+        'property_attributes': {
+          'logfeeder_external_solr_kerberos_keytab': {'visible': 'false'},
+          'logfeeder_external_solr_kerberos_principal': {'visible': 'false'}
+        }
+      },
+      'logsearch-common-env': {
+        'properties': {
+          'logsearch_use_external_solr': 'true',
+          'logsearch_external_solr_kerberos_enabled': 'false'
+        },
+        'property_attributes': {
+          'logsearch_external_solr_kerberos_enabled': {'visible': 'false'}
+        }
+      }
+    }
+    services = {
+      "services": [],
+      "configurations": {
+        "logsearch-properties": {
+          "properties": {
+            "logsearch.collection.numshards" : "5",
+            "logsearch.collection.replication.factor": "0"
           }
         }
-      ]
+      },
+      "changed-configurations": [ ]
+
     }
     def return_c6401_hostname(services, service_name, component_name):
-      return ["c6401.ambari.apache.org"]
+      if service_name == "LOGSEARCH" and component_name == "LOGSEARCH_SERVER":
+        return ["c6401.ambari.apache.org"]
+      else:
+        return []
     self.serviceAdvisor.getComponentHostNames = return_c6401_hostname
-    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
-    self.assertEquals(configurations, expected)
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.configurations, self.clusterData, services, self.hosts)
+    self.assertEquals(self.configurations, expected)