task(index): Allow replica auto-expansion to be configured and set a more intelligent default.

dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESIndexAPI.java

@@ -457,10 +457,18 @@ public synchronized CreateIndexStatus createIndex(final String indexName, String
 
 		shards = shards > 0 ? shards : Config.getIntProperty("es.index.number_of_shards", 1);
 
+		if(shards>1){
+			Logger.warn(this.getClass(),"Number of ES shards : " + shards + ". Important to note that the more shards you enable, the slower the index reads.  dotCMS recommends using only 1 shard per replica. ");
+		}
 		Map<String,Object> map  = (settings ==null) ? new HashMap<>() : new ObjectMapper().readValue(settings, LinkedHashMap.class);
 
+
+		String autoExpandReplicas = Config.getStringProperty("ES_INDEX_AUTO_EXPAND_REPLICAS", "0-1");
+
+
 		map.put("number_of_shards", shards);
-		map.put("index.auto_expand_replicas", "0-all");
+		map.put("index.auto_expand_replicas", autoExpandReplicas);
+
 		map.putIfAbsent("index.mapping.total_fields.limit", 10000);
 		map.putIfAbsent("index.mapping.nested_fields.limit", 10000);
 		map.putIfAbsent("index.query.default_field", Config.getStringProperty("ES_INDEX_QUERY_DEFAULT_FIELD", "catchall"));

dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSIndexAPIImpl.java

@@ -125,12 +125,17 @@ public CreateIndexStatus createIndex(final String indexName, String settings, in
             "Trying to create index: " + indexName + " with shards: " + shards);
 
         shards = shards > 0 ? shards : Config.getIntProperty("opensearch.index.number_of_shards", 1);
-
+        if(shards>1){
+            Logger.warn(this.getClass(),"Number of OS shards : " + shards + ". Important to note that the more shards you enable, the slower the index reads.  dotCMS recommends using only 1 shard per replica. ");
+        }
         Map<String, Object> settingsMap = (settings == null) ? new HashMap<>() :
             objectMapper.readValue(settings, LinkedHashMap.class);
 
+        String autoExpandReplicas = Config.getStringProperty("ES_INDEX_AUTO_EXPAND_REPLICAS", "0-1");
+
+
         settingsMap.put("index.number_of_shards", shards);
-        settingsMap.put("index.auto_expand_replicas", "0-all");
+        settingsMap.put("index.auto_expand_replicas", autoExpandReplicas);
         settingsMap.putIfAbsent("index.mapping.total_fields.limit", 10000);
         settingsMap.putIfAbsent("index.mapping.nested_fields.limit", 10000);
         settingsMap.putIfAbsent("index.query.default_field",
@@ -141,7 +146,7 @@ public CreateIndexStatus createIndex(final String indexName, String settings, in
             builder.index(getNameWithClusterIDPrefix(indexName))
                    .settings(IndexSettings.of(settingsBuilder -> {
                        settingsBuilder.numberOfShards(finalShards);
-                       settingsBuilder.autoExpandReplicas("0-all");
+                       settingsBuilder.autoExpandReplicas(autoExpandReplicas);
                        return settingsBuilder;
                    }))
                    .timeout(Time.of(timeBuilder ->
@@ -442,7 +447,7 @@ public String getDefaultIndexSettings() {
             settings = "{\n" +
                 "  \"number_of_shards\": 1,\n" +
                 "  \"number_of_replicas\": 0,\n" +
-                "  \"auto_expand_replicas\": \"0-all\",\n" +
+                "  \"auto_expand_replicas\": \"0-1\",\n" +
                 "  \"mapping\": {\n" +
                 "    \"total_fields\": { \"limit\": 10000 },\n" +
                 "    \"nested_fields\": { \"limit\": 10000 }\n" +

dotCMS/src/main/java/com/dotcms/rest/api/v1/index/ESIndexResource.java

@@ -324,10 +324,9 @@ public Response getReindexationProgress(@Context final HttpServletRequest reques
     @NoCache
     @Path("/reindex")
     @Produces({MediaType.APPLICATION_JSON})
-    public Response startReindex(@Context final HttpServletRequest request, @Context final HttpServletResponse response,
-                    @QueryParam("shards") int shards, @DefaultValue(DOTALL) @QueryParam("contentType") String contentType) throws DotDataException, DotSecurityException {
+    public Response startReindex(@Context final HttpServletRequest request, @Context final HttpServletResponse response, @DefaultValue(DOTALL) @QueryParam("contentType") String contentType) throws DotDataException, DotSecurityException {
         final InitDataObject init = auth(request, response);
-        shards = (shards <= 0) ? Config.getIntProperty("es.index.number_of_shards", 2) : shards;
+        int shards = Config.getIntProperty("es.index.number_of_shards", 1) ;
 
         System.setProperty("es.index.number_of_shards", String.valueOf(shards));
         Logger.info(this, "Running Contentlet Reindex");
@@ -344,7 +343,6 @@ public Response startReindex(@Context final HttpServletRequest request, @Context
 
         return getReindexationProgress(request, response);
 
-
     }
 
     @CloseDBIfOpened

dotCMS/src/main/java/com/dotmarketing/portlets/cmsmaintenance/action/ViewCMSMaintenanceAction.java

@@ -40,11 +40,6 @@
 import com.liferay.portlet.ActionResponseImpl;
 import com.liferay.util.FileUtil;
 import com.liferay.util.servlet.SessionMessages;
-import org.apache.commons.io.output.TeeOutputStream;
-import org.apache.commons.lang.StringUtils;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.servlet.jsp.PageContext;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -58,6 +53,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.zip.ZipOutputStream;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.jsp.PageContext;
+import org.apache.commons.io.output.TeeOutputStream;
+import org.apache.commons.lang.StringUtils;
 
 /**
  * This class group all the CMS Maintenance Task
@@ -142,19 +141,14 @@ public void processAction(
 				if(!InodeUtils.isSet(structure.getInode()))
 				{
 
-						int shards = Config.getIntProperty("es.index.number_of_shards", 2);
-						try{
-							shards = Integer.parseInt(req.getParameter("shards"));
-						}catch(Exception e){
+					int shards = Config.getIntProperty("es.index.number_of_shards", 1);
+					System.setProperty("es.index.number_of_shards", String.valueOf(shards));
+					Logger.info(this, "Running Contentlet Reindex");
 
-						}
-						System.setProperty("es.index.number_of_shards", String.valueOf(shards));
-						Logger.info(this, "Running Contentlet Reindex");
+					conAPI.refreshAllContent();
 
-						conAPI.refreshAllContent();
-
-						message = "message.cmsmaintenance.cache.indexrebuilt";
-						AdminLogger.log(ViewCMSMaintenanceAction.class, "processAction", "Running Contentlet Reindex");
+					message = "message.cmsmaintenance.cache.indexrebuilt";
+					AdminLogger.log(ViewCMSMaintenanceAction.class, "processAction", "Running Contentlet Reindex");
 
 				}
 				else

dotCMS/src/main/webapp/WEB-INF/openapi/openapi.yaml

@@ -8582,11 +8582,6 @@ paths:
     post:
       operationId: startReindex
       parameters:
-      - in: query
-        name: shards
-        schema:
-          type: integer
-          format: int32
       - in: query
         name: contentType
         schema:

dotCMS/src/main/webapp/html/portlet/ext/cmsmaintenance/index_stats.jsp

@@ -94,7 +94,6 @@ Map<String,ClusterIndexHealth> map = esapi.getClusterHealth();
 					<th><%= LanguageUtil.get(pageContext,"Index-Name") %></th>
 					<th><%= LanguageUtil.get(pageContext,"Created") %></th>
 					<th style="text-align: center"><%= LanguageUtil.get(pageContext,"Count") %></th>
-					<th style="text-align: center"><%= LanguageUtil.get(pageContext,"Shards") %></th>
 					<th style="text-align: center"><%= LanguageUtil.get(pageContext,"Replicas") %></th>
 					<th style="text-align: center"><%= LanguageUtil.get(pageContext,"Size") %></th>
 					<th style="text-align: center"><%= LanguageUtil.get(pageContext,"Health") %></th>
@@ -133,7 +132,6 @@ Map<String,ClusterIndexHealth> map = esapi.getClusterHealth();
 					<td align="center">
 						<%=status !=null ? status.documentCount() : "n/a"%>
 					</td>
-					<td align="center"><%=(health !=null) ? health.numberOfShards() : "n/a"%></td>
 					<td align="center"><%=(health !=null) ? health.numberOfReplicas(): "n/a"%></td>
 					<td align="center"><%=status !=null ? status.size(): "n/a"%></td>
 					<td align="center">
@@ -300,4 +298,4 @@ Map<String,ClusterIndexHealth> map = esapi.getClusterHealth();
 
 				</div>
 			<%} %>
-		--%>
+		--%>

dotCMS/src/main/webapp/html/portlet/ext/cmsmaintenance/view_cms_maintenance.jsp

@@ -215,19 +215,9 @@ function refreshIndexStats(){
 function doReindex(){
     var shards =1
     var contentType = dijit.byId('structure').item != null ? dijit.byId('structure').item.id : "DOTALL";
-    if("DOTALL"=== contentType){
-        var number=prompt("<%=UtilMethods.escapeDoubleQuotes(LanguageUtil.get(pageContext, "Number-of-Shards"))%> ", <%=Config.getIntProperty("es.index.number_of_shards", 2)%>);
-        if(!number){
-            return;
-        }
-        shards = parseInt(number);
-        if(shards == null || shards <1){
-            return;
-        }
-    }
     dijit.byId('structure').reset();
 
-    fetch('/api/v1/esindex/reindex?shards=' + shards + '&contentType=' + contentType, {method:'POST'})
+    fetch('/api/v1/esindex/reindex?contentType=' + contentType, {method: 'POST'})
     .then(response => response.json())
     .then(data =>checkReindexationCallback(data))
     .then(()=>refreshIndexStats());
@@ -376,7 +366,6 @@ const indexTableRowTmpl = (data) => `<tr class="${data.rowClass} showPointer" id
     <td  class="showPointer" >${data.indexName}</td>
     <td>${data.created}</td>
     <td align="center">${data.documentCount}</td>
-    <td align="center">${data.numberOfShards}</td>
     <td align="center">${data.numberOfReplicas}</td>
     <td align="center">${data.size}</td>
     <td align="center">
@@ -402,7 +391,7 @@ function paintStatusTable(data){
         data.created = item.created;
         data.indexColor=item.health.status;
         data.numberOfReplicas=item.health.numberOfReplicas;
-        data.numberOfShards=item.health.numberOfShards;
+        //data.numberOfShards=item.health.numberOfShards;
         data.documentCount=(item.status === undefined) ? "n/a"  : item.status.documentCount;
         data.size=(item.status === undefined) ? "n/a"  : item.status.size;
 
@@ -1329,7 +1318,8 @@ dd.leftdl {
         <input type="hidden" name="dataOnly" id="dataOnly">
         <input type="hidden" name="cmd" value="">
         <input type="hidden"  name="defaultStructure" id="defaultStructure" value="">
-        <input type="hidden" name="shards" id="numberOfShards" value="<%=Config.getIntProperty("es.index.number_of_shards", 2)%>">
+        <input type="hidden" name="shards" id="numberOfShards"
+               value="<%=Config.getIntProperty("es.index.number_of_shards", 1)%>">
 
         <!-- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
         <!-- START Cache TAB -->
@@ -1980,7 +1970,7 @@ dd.leftdl {
 
  	<div align="center" style="padding-top: 10px;">
 		<label name="index"><%=UtilMethods.escapeDoubleQuotes(LanguageUtil.get(pageContext, "Number-of-Shards"))%></label>
-  		<input type="text" id="shards" name="shards" value="<%=Config.getIntProperty("es.index.number_of_shards", 2)%>">
+        <input type="text" id="shards" name="shards" value="<%=Config.getIntProperty("es.index.number_of_shards", 1)%>">
   	</div><br />
   	<div class="buttonRow" align="center">
 		<button dojoType="dijit.form.Button" iconClass="cancelIcon" onClick="javascript:dijit.byId('addIndex').hide();"><%= LanguageUtil.get(pageContext, "Cancel") %></button>&nbsp; &nbsp;

dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESIndexAPITest.java

@@ -62,7 +62,7 @@ public void test_createIndex_newIndexShouldHaveProperReplicasSetting() throws IO
 
             String replicasSetting = getSettingsResponse.getSetting(fullNewIndexName, "index.auto_expand_replicas");
 
-            Assert.assertEquals("0-all", replicasSetting);
+            Assert.assertEquals("0-1", replicasSetting);
         } finally {
             esIndexAPI.delete(esIndexAPI.getNameWithClusterIDPrefix(newIndexName));
         }

dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSIndexAPIImplIntegrationTest.java

@@ -323,7 +323,7 @@ public void test_osAndVersionedRegistry_divergence_shouldBeDetectable() throws E
 
     /**
      * Given scenario: A new index is created via {@link OSIndexAPIImpl#createIndex(String, int)}
-     * Expected: The OpenSearch cluster stores {@code auto_expand_replicas=0-all} on that index,
+     * Expected: The OpenSearch cluster stores {@code auto_expand_replicas=0-1} on that index,
      *           consistent with the behaviour verified in {@code ESIndexAPITest}.
      */
     @Test
@@ -336,7 +336,7 @@ public void test_createIndex_shouldHaveAutoExpandReplicasSetting() throws Except
 
         assertNotNull("Index state must be present in GET response", response.result().get(fullName));
         final String autoExpand = response.result().get(fullName).settings().index().autoExpandReplicas();
-        assertEquals("auto_expand_replicas must be 0-all on a freshly created index", "0-all", autoExpand);
+        assertEquals("auto_expand_replicas must be 0-1 on a freshly created index", "0-1", autoExpand);
         Logger.info(this, "✅ test_createIndex_shouldHaveAutoExpandReplicasSetting passed");
     }
 
@@ -557,4 +557,4 @@ private void cleanupVersionedRows() {
             Logger.warn(this, "Cleanup: error removing versioned DB rows: " + e.getMessage());
         }
     }
-}
+}