From dbb0a7955c67529483435cb57581b082a5bdd432 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 13 Jan 2026 17:04:05 +0530 Subject: [PATCH 01/14] fix: optimize the index and reduce the size --- .../ElasticsearchIndexingService.java | 97 +++- .../elasticsearch/ElasticsearchService.java | 442 +++++++----------- src/main/resources/application.properties | 34 ++ 3 files changed, 290 insertions(+), 283 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java index 941a055..458dedf 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java @@ -3,6 +3,9 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.mapping.*; import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; +import co.elastic.clients.elasticsearch.indices.IndexSettings; +import co.elastic.clients.elasticsearch.indices.TranslogDurability; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,7 +30,7 @@ public class ElasticsearchIndexingService { private String beneficiaryIndex; /** - * Create or recreate the Elasticsearch index with proper mapping + */ public void createIndexWithMapping() throws Exception { logger.info("Creating index with mapping: {}", beneficiaryIndex); @@ -38,24 +41,71 @@ public void createIndexWithMapping() throws Exception { esClient.indices().delete(d -> d.index(beneficiaryIndex)); } - // Create index with mapping + IndexSettings settings = IndexSettings.of(s -> s + .refreshInterval(t -> t.time("30s")) + + .numberOfShards("3") + + .numberOfReplicas("1") + + .queries(q -> q + .cache(c -> c.enabled(true)) + ) + + .maxResultWindow(10000) + + .translog(t -> t + .durability(TranslogDurability.Async) + .syncInterval(ts -> ts.time("30s")) + ) + ); + TypeMapping mapping = TypeMapping.of(tm -> tm .properties("benId", Property.of(p -> p.keyword(k -> k))) .properties("benRegId", Property.of(p -> p.long_(l -> l))) + .properties("beneficiaryID", Property.of(p -> p.keyword(k -> k))) + .properties("firstName", Property.of(p -> p.text(t -> t - .fields("keyword", Property.of(fp -> fp.keyword(k -> k)))))) + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + .fields("prefix", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) // Fast prefix search + ))) + ))) + .properties("lastName", Property.of(p -> p.text(t -> t - .fields("keyword", Property.of(fp -> fp.keyword(k -> k)))))) + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + .fields("prefix", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) + ))) + ))) + + .properties("fatherName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + ))) + + .properties("spouseName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + ))) + .properties("genderID", Property.of(p -> p.integer(i -> i))) .properties("genderName", Property.of(p -> p.keyword(k -> k))) - .properties("dOB", Property.of(p -> p.date(d -> d))) + .properties("dOB", Property.of(p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis")))) .properties("age", Property.of(p -> p.integer(i -> i))) - .properties("phoneNum", Property.of(p -> p.keyword(k -> k))) - .properties("fatherName", Property.of(p -> p.text(t -> t - .fields("keyword", Property.of(fp -> fp.keyword(k -> k)))))) - .properties("spouseName", Property.of(p -> p.text(t -> t - .fields("keyword", Property.of(fp -> fp.keyword(k -> k)))))) + + .properties("phoneNum", Property.of(p -> p.keyword(k -> k + .fields("ngram", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .searchAnalyzer("standard") + ))) + ))) + .properties("isHIVPos", Property.of(p -> p.keyword(k -> k))) .properties("createdBy", Property.of(p -> p.keyword(k -> k))) .properties("createdDate", Property.of(p -> p.date(d -> d))) @@ -94,15 +144,37 @@ public void createIndexWithMapping() throws Exception { esClient.indices().create(c -> c .index(beneficiaryIndex) + .settings(settings) .mappings(mapping) ); logger.info("Index created successfully: {}", beneficiaryIndex); } + /** + * Reset refresh interval after bulk indexing completes + * Call this after syncAllBeneficiaries() finishes + */ + public void optimizeForSearch() throws Exception { + logger.info("Optimizing index for search performance..."); + + esClient.indices().putSettings(s -> s + .index(beneficiaryIndex) + .settings(is -> is + .refreshInterval(t -> t.time("1s")) + .translog(t -> t.durability(TranslogDurability.Request)) + ) + ); + + esClient.indices().forcemerge(f -> f + .index(beneficiaryIndex) + .maxNumSegments(1L) // Optimal for read-heavy workloads + ); + + } + /** * Index all beneficiaries - delegates to existing sync service - * This is much safer than loading all records at once */ public Map indexAllBeneficiaries() { logger.info("Starting full indexing via sync service..."); @@ -110,6 +182,9 @@ public Map indexAllBeneficiaries() { try { ElasticsearchSyncService.SyncResult result = syncService.syncAllBeneficiaries(); + // After indexing completes, optimize for search + optimizeForSearch(); + Map response = new HashMap<>(); response.put("success", result.getSuccessCount()); response.put("failed", result.getFailureCount()); diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index a8b10ae..89fa591 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -20,14 +20,10 @@ import org.springframework.stereotype.Service; import java.math.BigDecimal; -import java.math.BigInteger; import java.sql.Timestamp; import java.util.*; import java.util.stream.Collectors; import co.elastic.clients.elasticsearch._types.SortOrder; -import co.elastic.clients.elasticsearch._types.query_dsl.FunctionScore; -import co.elastic.clients.elasticsearch._types.query_dsl.FunctionScoreMode; -import co.elastic.clients.elasticsearch._types.query_dsl.FunctionBoostMode; import com.iemr.common.identity.repo.BenAddressRepo; @Service @@ -58,82 +54,96 @@ public class ElasticsearchService { public List> universalSearch(String query, Integer userId) { try { final Map userLocation = - (userId != null) ? getUserLocation(userId) : null; - + (userId != null) ? getUserLocation(userId) : null; boolean isNumeric = query.matches("\\d+"); - - // Determine minimum score threshold based on query type - double minScore = isNumeric ? 1.0 : 3.0; + double minScore = isNumeric ? 1.0 : 2.0; SearchResponse response = esClient.search(s -> s .index(beneficiaryIndex) + + .preference("_local") + + .requestCache(true) + .query(q -> q .functionScore(fs -> fs .query(qq -> qq .bool(b -> { if (!isNumeric) { - // Name searches with higher boost for exact matches + // OPTIMIZED NAME SEARCH + // Use match_phrase_prefix for faster prefix matching b.should(s1 -> s1.multiMatch(mm -> mm .query(query) - .fields("firstName^3", "lastName^3", "fatherName^2", "spouseName^2") - .type(TextQueryType.BestFields) - .fuzziness("AUTO") + .fields("firstName^3", "lastName^3") + .type(TextQueryType.Phrase) + .boost(3.0f) + )); + + b.should(s2 -> s2.term(t -> t + .field("firstName.keyword") + .value(query) + .boost(10.0f) + )); + b.should(s3 -> s3.term(t -> t + .field("lastName.keyword") + .value(query) + .boost(10.0f) + )); + + // Prefix search using index_prefixes (FAST!) + b.should(s4 -> s4.match(m -> m + .field("firstName.prefix") + .query(query) + .boost(2.0f) + )); + b.should(s5 -> s5.match(m -> m + .field("lastName.prefix") + .query(query) + .boost(2.0f) )); - - // Exact keyword matches get highest boost - b.should(s2 -> s2.term(t -> t.field("firstName.keyword").value(query).boost(5.0f))); - b.should(s3 -> s3.term(t -> t.field("lastName.keyword").value(query).boost(5.0f))); - b.should(s4 -> s4.term(t -> t.field("fatherName.keyword").value(query).boost(4.0f))); - b.should(s5 -> s5.term(t -> t.field("spouseName.keyword").value(query).boost(4.0f))); } - // ID searches - exact matches get very high boost - b.should(s6 -> s6.term(t -> t.field("healthID").value(query).boost(10.0f))); - b.should(s7 -> s7.term(t -> t.field("abhaID").value(query).boost(10.0f))); - b.should(s8 -> s8.term(t -> t.field("familyID").value(query).boost(8.0f))); - b.should(s9 -> s9.term(t -> t.field("beneficiaryID").value(query).boost(10.0f))); - b.should(s10 -> s10.term(t -> t.field("benId").value(query).boost(10.0f))); - b.should(s11 -> s11.term(t -> t.field("aadharNo").value(query).boost(9.0f))); - b.should(s12 -> s12.term(t -> t.field("govtIdentityNo").value(query).boost(8.0f))); + b.should(s6 -> s6.term(t -> t.field("healthID").value(query).boost(15.0f))); + b.should(s7 -> s7.term(t -> t.field("abhaID").value(query).boost(15.0f))); + b.should(s8 -> s8.term(t -> t.field("beneficiaryID").value(query).boost(15.0f))); + b.should(s9 -> s9.term(t -> t.field("benId").value(query).boost(15.0f))); + b.should(s10 -> s10.term(t -> t.field("aadharNo").value(query).boost(12.0f))); if (isNumeric) { - // Partial matches for numeric fields - b.should(s13 -> s13.wildcard(w -> w.field("phoneNum").value("*" + query + "*").boost(3.0f))); - b.should(s14 -> s14.wildcard(w -> w.field("healthID").value("*" + query + "*").boost(2.0f))); - b.should(s15 -> s15.wildcard(w -> w.field("abhaID").value("*" + query + "*").boost(2.0f))); - b.should(s16 -> s16.wildcard(w -> w.field("familyID").value("*" + query + "*").boost(2.0f))); - b.should(s17 -> s17.wildcard(w -> w.field("beneficiaryID").value("*" + query + "*").boost(2.0f))); - b.should(s18 -> s18.wildcard(w -> w.field("benId").value("*" + query + "*").boost(2.0f))); - b.should(s19 -> s19.wildcard(w -> w.field("aadharNo").value("*" + query + "*").boost(2.0f))); - b.should(s20 -> s20.wildcard(w -> w.field("govtIdentityNo").value("*" + query + "*").boost(2.0f))); + // PREFIX QUERIES (much faster than wildcard) + b.should(s11 -> s11.prefix(p -> p.field("phoneNum").value(query).boost(5.0f))); + b.should(s12 -> s12.prefix(p -> p.field("healthID").value(query).boost(4.0f))); + b.should(s13 -> s13.prefix(p -> p.field("abhaID").value(query).boost(4.0f))); + b.should(s14 -> s14.prefix(p -> p.field("beneficiaryID").value(query).boost(4.0f))); + + // ONLY use wildcard if query is long enough (>= 4 digits) + if (query.length() >= 4) { + b.should(s15 -> s15.wildcard(w -> w + .field("phoneNum") + .value("*" + query + "*") + .boost(2.0f) + )); + } - // Prefix matches - b.should(s21 -> s21.prefix(p -> p.field("phoneNum").value(query).boost(4.0f))); - b.should(s22 -> s22.prefix(p -> p.field("healthID").value(query).boost(3.0f))); - b.should(s23 -> s23.prefix(p -> p.field("abhaID").value(query).boost(3.0f))); - b.should(s24 -> s24.prefix(p -> p.field("familyID").value(query).boost(3.0f))); - b.should(s25 -> s25.prefix(p -> p.field("beneficiaryID").value(query).boost(3.0f))); - b.should(s26 -> s26.prefix(p -> p.field("benId").value(query).boost(3.0f))); - try { Long numericValue = Long.parseLong(query); - b.should(s27 -> s27.term(t -> t.field("benRegId").value(numericValue).boost(10.0f))); - b.should(s28 -> s28.term(t -> t.field("benAccountID").value(numericValue).boost(8.0f))); + b.should(s16 -> s16.term(t -> t.field("benRegId").value(numericValue).boost(15.0f))); + b.should(s17 -> s17.term(t -> t.field("benAccountID").value(numericValue).boost(10.0f))); int intValue = numericValue.intValue(); - b.should(s29 -> s29.term(t -> t.field("genderID").value(intValue).boost(2.0f))); - b.should(s30 -> s30.term(t -> t.field("age").value(intValue).boost(1.0f))); - b.should(s31 -> s31.term(t -> t.field("stateID").value(intValue).boost(1.0f))); - b.should(s32 -> s32.term(t -> t.field("districtID").value(intValue).boost(1.0f))); - b.should(s33 -> s33.term(t -> t.field("blockID").value(intValue).boost(1.0f))); - b.should(s34 -> s34.term(t -> t.field("villageID").value(intValue).boost(1.0f))); - b.should(s35 -> s35.term(t -> t.field("servicePointID").value(intValue).boost(1.0f))); - b.should(s36 -> s36.term(t -> t.field("parkingPlaceID").value(intValue).boost(1.0f))); - - logger.info("Added numeric searches for value: {}", numericValue); + if (userLocation != null) { + Integer userVillageId = userLocation.get("villageId"); + Integer userBlockId = userLocation.get("blockId"); + + if (userVillageId != null && userVillageId == intValue) { + b.should(s18 -> s18.term(t -> t.field("villageID").value(intValue).boost(3.0f))); + } + if (userBlockId != null && userBlockId == intValue) { + b.should(s19 -> s19.term(t -> t.field("blockID").value(intValue).boost(2.0f))); + } + } } catch (NumberFormatException e) { - logger.warn("Failed to parse numeric value: {}", query); } } @@ -141,51 +151,55 @@ public List> universalSearch(String query, Integer userId) { return b; }) ) - // Add location-based scoring if user location is available .functions(getFunctionScores(userLocation)) .scoreMode(FunctionScoreMode.Sum) .boostMode(FunctionBoostMode.Multiply) + .maxBoost(5.0) ) ) - .minScore(minScore) - .size(500) - .sort(so -> so - .score(sc -> sc.order(SortOrder.Desc)) + .minScore(minScore) + + .size(100) // Reduced from 500 + + .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) + + .source(src -> src + .filter(f -> f + .includes("benRegId", "beneficiaryID", "firstName", "lastName", + "genderID", "genderName", "dOB", "phoneNum", + "stateID", "districtID", "blockID", "villageID") + ) ) + + , BeneficiariesESDTO.class); - logger.info("ES returned {} hits for query: '{}' (min score: {})", - response.hits().hits().size(), query, minScore); + logger.info("ES returned {} hits in {}ms for query: '{}'", + response.hits().hits().size(), + response.took(), + query); + + if (response.hits().hits().isEmpty()) { + logger.info("No results in ES, using database fallback"); + return searchInDatabaseDirectly(query); + } - List> allResults = response.hits().hits().stream() + List> results = response.hits().hits().stream() .map(hit -> { - if (hit.source() != null) { - logger.debug("Hit score: {}, benRegId: {}, name: {} {}", - hit.score(), - hit.source().getBenRegId(), - hit.source().getFirstName(), - hit.source().getLastName()); - } Map result = mapESResultToExpectedFormat(hit.source()); if (result != null) { - result.put("_score", hit.score()); + result.put("_score", hit.score()); } return result; }) .filter(Objects::nonNull) .collect(Collectors.toList()); - if (allResults.isEmpty()) { - logger.info("No results found in ES, falling back to database"); - return searchInDatabaseDirectly(query); - } - - logger.info("Returning {} matched results", allResults.size()); - return allResults; + logger.info("Returning {} results", results.size()); + return results; } catch (Exception e) { - logger.error("ES universal search failed: {}", e.getMessage(), e); - logger.info("Fallback: Searching in MySQL database"); + logger.error("ES search failed: {}", e.getMessage()); return searchInDatabaseDirectly(query); } } @@ -203,19 +217,19 @@ private List getFunctionScores(Map userLocation) Integer userVillageId = userLocation.get("villageId"); Integer userBlockId = userLocation.get("blockId"); - // Village match - highest boost + // Village match if (userVillageId != null) { scores.add(FunctionScore.of(f -> f .filter(ff -> ff.term(t -> t.field("villageID").value(userVillageId))) - .weight(3.0) + .weight(2.0) )); } - // Block match - medium boost + // Block match if (userBlockId != null) { scores.add(FunctionScore.of(f -> f .filter(ff -> ff.term(t -> t.field("blockID").value(userBlockId))) - .weight(2.0) + .weight(1.5) )); } @@ -223,7 +237,7 @@ private List getFunctionScores(Map userLocation) } /** - * Advanced search with multiple criteria - only returns actual matches + * Advanced search with filter context */ public List> advancedSearch( String firstName, @@ -243,209 +257,93 @@ public List> advancedSearch( Integer userId) { try { - logger.info("ES Advanced Search - firstName: {}, lastName: {}, genderId: {}, stateId: {}, districtId: {}, blockId: {}, villageId: {}", - firstName, lastName, genderId, stateId, districtId, blockId, villageId); - final Map userLocation = - (userId != null) ? getUserLocation(userId) : null; - + (userId != null) ? getUserLocation(userId) : null; SearchResponse response = esClient.search(s -> s .index(beneficiaryIndex) + .preference("_local") + .requestCache(true) + .query(q -> q - .functionScore(fs -> fs - .query(qq -> qq - .bool(b -> { - // Name searches with fuzzy matching and boost - if (firstName != null && !firstName.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.match(mm -> mm - .field("firstName") - .query(firstName) - .fuzziness("AUTO") - .boost(2.0f) - )) - .should(s2 -> s2.term(t -> t - .field("firstName.keyword") - .value(firstName) - .boost(5.0f) - )) - .minimumShouldMatch("1") - )); - } - - if (lastName != null && !lastName.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.match(mm -> mm - .field("lastName") - .query(lastName) - .fuzziness("AUTO") - .boost(2.0f) - )) - .should(s2 -> s2.term(t -> t - .field("lastName.keyword") - .value(lastName) - .boost(5.0f) - )) - .minimumShouldMatch("1") - )); - } - - if (fatherName != null && !fatherName.trim().isEmpty()) { - b.must(m -> m.match(mm -> mm - .field("fatherName") - .query(fatherName) - .fuzziness("AUTO") - .boost(2.0f) - )); - } - - if (spouseName != null && !spouseName.trim().isEmpty()) { - b.must(m -> m.match(mm -> mm - .field("spouseName") - .query(spouseName) - .fuzziness("AUTO") - .boost(2.0f) - )); - } - - // Exact matches for IDs and structured data - if (genderId != null) { - b.filter(m -> m.term(t -> t - .field("genderID") - .value(genderId) - )); - } - - if (dob != null) { - b.must(m -> m.term(t -> t - .field("dob") - .value(dob.getTime()) - .boost(5.0f) - )); - } - - // Location filters - if (stateId != null) { - b.filter(m -> m.term(t -> t - .field("stateID") - .value(stateId) - )); - } - - if (districtId != null) { - b.filter(m -> m.term(t -> t - .field("districtID") - .value(districtId) - )); - } - - if (blockId != null) { - b.filter(m -> m.term(t -> t - .field("blockID") - .value(blockId) - )); - } - - if (villageId != null) { - b.must(m -> m.term(t -> t - .field("villageID") - .value(villageId) - .boost(3.0f) - )); - } - - // Identity searches - if (phoneNumber != null && !phoneNumber.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.term(t -> t - .field("phoneNum") - .value(phoneNumber) - .boost(5.0f) - )) - .should(s2 -> s2.wildcard(w -> w - .field("phoneNum") - .value("*" + phoneNumber + "*") - .boost(2.0f) - )) - .minimumShouldMatch("1") - )); - } - - if (beneficiaryId != null && !beneficiaryId.trim().isEmpty()) { - b.must(m -> m.term(t -> t - .field("beneficiaryID") - .value(beneficiaryId) - .boost(10.0f) - )); - } - - if (healthId != null && !healthId.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.term(t -> t - .field("healthID") - .value(healthId) - .boost(10.0f) - )) - .should(s2 -> s2.term(t -> t - .field("abhaID") - .value(healthId) - .boost(10.0f) - )) - .minimumShouldMatch("1") - )); - } - - if (aadharNo != null && !aadharNo.trim().isEmpty()) { - b.must(m -> m.term(t -> t - .field("aadharNo") - .value(aadharNo) - .boost(10.0f) - )); - } - - return b; - }) - ) - // Add location-based scoring - .functions(getFunctionScores(userLocation)) - .scoreMode(FunctionScoreMode.Sum) - .boostMode(FunctionBoostMode.Multiply) - ) - ) - .minScore(2.0) - .size(500) - .sort(so -> so - .score(sc -> sc.order(SortOrder.Desc)) + .bool(b -> { + // Use FILTER context for exact matches (faster, cached) + if (genderId != null) { + b.filter(f -> f.term(t -> t.field("genderID").value(genderId))); + } + if (stateId != null) { + b.filter(f -> f.term(t -> t.field("stateID").value(stateId))); + } + if (districtId != null) { + b.filter(f -> f.term(t -> t.field("districtID").value(districtId))); + } + if (blockId != null) { + b.filter(f -> f.term(t -> t.field("blockID").value(blockId))); + } + if (villageId != null) { + b.filter(f -> f.term(t -> t.field("villageID").value(villageId))); + } + + // MUST context for scored searches + if (firstName != null && !firstName.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1.term(t -> t.field("firstName.keyword").value(firstName).boost(5.0f))) + .should(s2 -> s2.match(mm -> mm.field("firstName").query(firstName).boost(2.0f))) + .minimumShouldMatch("1") + )); + } + + if (lastName != null && !lastName.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1.term(t -> t.field("lastName.keyword").value(lastName).boost(5.0f))) + .should(s2 -> s2.match(mm -> mm.field("lastName").query(lastName).boost(2.0f))) + .minimumShouldMatch("1") + )); + } + + // Exact match IDs in filter context + if (beneficiaryId != null && !beneficiaryId.trim().isEmpty()) { + b.filter(f -> f.term(t -> t.field("beneficiaryID").value(beneficiaryId))); + } + if (healthId != null && !healthId.trim().isEmpty()) { + b.filter(f -> f.bool(bb -> bb + .should(s1 -> s1.term(t -> t.field("healthID").value(healthId))) + .should(s2 -> s2.term(t -> t.field("abhaID").value(healthId))) + .minimumShouldMatch("1") + )); + } + if (aadharNo != null && !aadharNo.trim().isEmpty()) { + b.filter(f -> f.term(t -> t.field("aadharNo").value(aadharNo))); + } + if (phoneNumber != null && !phoneNumber.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1.term(t -> t.field("phoneNum").value(phoneNumber).boost(3.0f))) + .should(s2 -> s2.prefix(p -> p.field("phoneNum").value(phoneNumber).boost(2.0f))) + .minimumShouldMatch("1") + )); + } + + return b; + }) ) + .size(100) + .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) + , BeneficiariesESDTO.class); - logger.info("ES advanced search returned {} hits", response.hits().hits().size()); - if (response.hits().hits().isEmpty()) { - logger.info("No results in ES, falling back to database"); return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, stateId, districtId, blockId, villageId, fatherName, spouseName, phoneNumber, beneficiaryId, healthId, aadharNo); } - List> results = response.hits().hits().stream() - .map(hit -> { - Map result = mapESResultToExpectedFormat(hit.source()); - if (result != null) { - result.put("_score", hit.score()); // Include score - } - return result; - }) + return response.hits().hits().stream() + .map(hit -> mapESResultToExpectedFormat(hit.source())) .filter(Objects::nonNull) .collect(Collectors.toList()); - logger.info("Returning {} matched results", results.size()); - return results; - } catch (Exception e) { - logger.error("ES advanced search failed: {}", e.getMessage(), e); - logger.info("Fallback: Searching in MySQL database"); + logger.error("ES advanced search failed: {}", e.getMessage()); return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, stateId, districtId, blockId, villageId, fatherName, spouseName, phoneNumber, beneficiaryId, healthId, aadharNo); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 22a9ff6..8774780 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -78,6 +78,40 @@ jwt.refresh.expiration=604800000 # Connection pool settings elasticsearch.connection.timeout=5000 elasticsearch.socket.timeout=60000 +elasticsearch.max.retry.timeout=60000 + +# Connection pooling +elasticsearch.max.connections=100 +elasticsearch.max.connections.per.route=50 + +# Request Configuration +elasticsearch.request.timeout=30000 +elasticsearch.max.result.window=10000 + +# Bulk Indexing Performance +elasticsearch.bulk.size=100 +elasticsearch.bulk.concurrent.requests=4 +elasticsearch.bulk.flush.interval=10s + +# Search Performance +elasticsearch.search.default.size=100 +elasticsearch.search.max.size=500 +elasticsearch.search.timeout=5s + +# Query Cache Settings +elasticsearch.query.cache.enabled=true +elasticsearch.query.cache.size=10% + +# Request Cache Settings +elasticsearch.request.cache.enabled=true + +# Circuit Breaker (Prevent OOM) +elasticsearch.circuit.breaker.enabled=true +elasticsearch.circuit.breaker.limit=95% + +# Thread Pool for Async Operations +elasticsearch.async.thread.pool.size=10 +elasticsearch.async.thread.pool.queue.size=1000 # Logging logging.level.com.iemr.common.identity.service.elasticsearch=INFO From 50d123c1828f2f2ef261c8532ae258f0375588b1 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 13 Jan 2026 20:01:09 +0530 Subject: [PATCH 02/14] fix: align indent --- .../elasticsearch/ElasticsearchService.java | 835 +++++++++--------- 1 file changed, 420 insertions(+), 415 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index 89fa591..430392c 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -7,10 +7,7 @@ import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType; import co.elastic.clients.elasticsearch.core.SearchResponse; -import com.iemr.common.identity.domain.Address; -import com.iemr.common.identity.dto.BeneficiariesDTO; import com.iemr.common.identity.dto.BeneficiariesESDTO; -import com.iemr.common.identity.dto.IdentitySearchDTO; import com.iemr.common.identity.repo.BenDetailRepo; import org.slf4j.Logger; @@ -28,354 +25,349 @@ @Service public class ElasticsearchService { - + private static final Logger logger = LoggerFactory.getLogger(ElasticsearchService.class); - + @Autowired private ElasticsearchClient esClient; - + @Autowired private BenDetailRepo benDetailRepo; @Autowired private BenAddressRepo benAddressRepo; - + @Value("${elasticsearch.index.beneficiary}") private String beneficiaryIndex; - + @Value("${elasticsearch.enabled}") private boolean esEnabled; - -/** - * Universal search with score-based filtering and location ranking - * Only returns records that actually match the query (not all 10000) - */ -public List> universalSearch(String query, Integer userId) { - try { - final Map userLocation = - (userId != null) ? getUserLocation(userId) : null; - - boolean isNumeric = query.matches("\\d+"); - double minScore = isNumeric ? 1.0 : 2.0; - - SearchResponse response = esClient.search(s -> s - .index(beneficiaryIndex) - - .preference("_local") - - .requestCache(true) - - .query(q -> q - .functionScore(fs -> fs - .query(qq -> qq - .bool(b -> { - if (!isNumeric) { - // OPTIMIZED NAME SEARCH - // Use match_phrase_prefix for faster prefix matching - b.should(s1 -> s1.multiMatch(mm -> mm - .query(query) - .fields("firstName^3", "lastName^3") - .type(TextQueryType.Phrase) - .boost(3.0f) - )); - - b.should(s2 -> s2.term(t -> t - .field("firstName.keyword") - .value(query) - .boost(10.0f) - )); - b.should(s3 -> s3.term(t -> t - .field("lastName.keyword") - .value(query) - .boost(10.0f) - )); - - // Prefix search using index_prefixes (FAST!) - b.should(s4 -> s4.match(m -> m - .field("firstName.prefix") - .query(query) - .boost(2.0f) - )); - b.should(s5 -> s5.match(m -> m - .field("lastName.prefix") - .query(query) - .boost(2.0f) - )); - } - - b.should(s6 -> s6.term(t -> t.field("healthID").value(query).boost(15.0f))); - b.should(s7 -> s7.term(t -> t.field("abhaID").value(query).boost(15.0f))); - b.should(s8 -> s8.term(t -> t.field("beneficiaryID").value(query).boost(15.0f))); - b.should(s9 -> s9.term(t -> t.field("benId").value(query).boost(15.0f))); - b.should(s10 -> s10.term(t -> t.field("aadharNo").value(query).boost(12.0f))); - - if (isNumeric) { - // PREFIX QUERIES (much faster than wildcard) - b.should(s11 -> s11.prefix(p -> p.field("phoneNum").value(query).boost(5.0f))); - b.should(s12 -> s12.prefix(p -> p.field("healthID").value(query).boost(4.0f))); - b.should(s13 -> s13.prefix(p -> p.field("abhaID").value(query).boost(4.0f))); - b.should(s14 -> s14.prefix(p -> p.field("beneficiaryID").value(query).boost(4.0f))); - - // ONLY use wildcard if query is long enough (>= 4 digits) - if (query.length() >= 4) { - b.should(s15 -> s15.wildcard(w -> w - .field("phoneNum") - .value("*" + query + "*") - .boost(2.0f) - )); - } - - try { - Long numericValue = Long.parseLong(query); - b.should(s16 -> s16.term(t -> t.field("benRegId").value(numericValue).boost(15.0f))); - b.should(s17 -> s17.term(t -> t.field("benAccountID").value(numericValue).boost(10.0f))); - - int intValue = numericValue.intValue(); - if (userLocation != null) { - Integer userVillageId = userLocation.get("villageId"); - Integer userBlockId = userLocation.get("blockId"); - - if (userVillageId != null && userVillageId == intValue) { - b.should(s18 -> s18.term(t -> t.field("villageID").value(intValue).boost(3.0f))); - } - if (userBlockId != null && userBlockId == intValue) { - b.should(s19 -> s19.term(t -> t.field("blockID").value(intValue).boost(2.0f))); - } - } - } catch (NumberFormatException e) { - } - } - - b.minimumShouldMatch("1"); - return b; - }) - ) - .functions(getFunctionScores(userLocation)) - .scoreMode(FunctionScoreMode.Sum) - .boostMode(FunctionBoostMode.Multiply) - .maxBoost(5.0) - ) - ) - .minScore(minScore) - - .size(100) // Reduced from 500 - - .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) - - .source(src -> src - .filter(f -> f - .includes("benRegId", "beneficiaryID", "firstName", "lastName", - "genderID", "genderName", "dOB", "phoneNum", - "stateID", "districtID", "blockID", "villageID") - ) - ) - - - , BeneficiariesESDTO.class); - - logger.info("ES returned {} hits in {}ms for query: '{}'", - response.hits().hits().size(), - response.took(), - query); - - if (response.hits().hits().isEmpty()) { - logger.info("No results in ES, using database fallback"); + /** + * Universal search with score-based filtering and location ranking + * Only returns records that actually match the query (not all 10000) + */ + public List> universalSearch(String query, Integer userId) { + try { + final Map userLocation = (userId != null) ? getUserLocation(userId) : null; + + boolean isNumeric = query.matches("\\d+"); + double minScore = isNumeric ? 1.0 : 2.0; + + SearchResponse response = esClient.search(s -> s + .index(beneficiaryIndex) + + .preference("_local") + + .requestCache(true) + + .query(q -> q + .functionScore(fs -> fs + .query(qq -> qq + .bool(b -> { + if (!isNumeric) { + // OPTIMIZED NAME SEARCH + // Use match_phrase_prefix for faster prefix matching + b.should(s1 -> s1.multiMatch(mm -> mm + .query(query) + .fields("firstName^3", "lastName^3") + .type(TextQueryType.Phrase) + .boost(3.0f))); + + b.should(s2 -> s2.term(t -> t + .field("firstName.keyword") + .value(query) + .boost(10.0f))); + b.should(s3 -> s3.term(t -> t + .field("lastName.keyword") + .value(query) + .boost(10.0f))); + + // Prefix search using index_prefixes (FAST!) + b.should(s4 -> s4.match(m -> m + .field("firstName.prefix") + .query(query) + .boost(2.0f))); + b.should(s5 -> s5.match(m -> m + .field("lastName.prefix") + .query(query) + .boost(2.0f))); + } + + b.should(s6 -> s6 + .term(t -> t.field("healthID").value(query).boost(15.0f))); + b.should(s7 -> s7 + .term(t -> t.field("abhaID").value(query).boost(15.0f))); + b.should(s8 -> s8 + .term(t -> t.field("beneficiaryID").value(query).boost(15.0f))); + b.should( + s9 -> s9.term(t -> t.field("benId").value(query).boost(15.0f))); + b.should(s10 -> s10 + .term(t -> t.field("aadharNo").value(query).boost(12.0f))); + + if (isNumeric) { + // PREFIX QUERIES (much faster than wildcard) + b.should(s11 -> s11 + .prefix(p -> p.field("phoneNum").value(query).boost(5.0f))); + b.should(s12 -> s12 + .prefix(p -> p.field("healthID").value(query).boost(4.0f))); + b.should(s13 -> s13 + .prefix(p -> p.field("abhaID").value(query).boost(4.0f))); + b.should(s14 -> s14.prefix( + p -> p.field("beneficiaryID").value(query).boost(4.0f))); + + // ONLY use wildcard if query is long enough (>= 4 digits) + if (query.length() >= 4) { + b.should(s15 -> s15.wildcard(w -> w + .field("phoneNum") + .value("*" + query + "*") + .boost(2.0f))); + } + + try { + Long numericValue = Long.parseLong(query); + b.should(s16 -> s16.term(t -> t.field("benRegId") + .value(numericValue).boost(15.0f))); + b.should(s17 -> s17.term(t -> t.field("benAccountID") + .value(numericValue).boost(10.0f))); + + int intValue = numericValue.intValue(); + if (userLocation != null) { + Integer userVillageId = userLocation.get("villageId"); + Integer userBlockId = userLocation.get("blockId"); + + if (userVillageId != null && userVillageId == intValue) { + b.should(s18 -> s18.term(t -> t.field("villageID") + .value(intValue).boost(3.0f))); + } + if (userBlockId != null && userBlockId == intValue) { + b.should(s19 -> s19.term(t -> t.field("blockID") + .value(intValue).boost(2.0f))); + } + } + } catch (NumberFormatException e) { + } + } + + b.minimumShouldMatch("1"); + return b; + })) + .functions(getFunctionScores(userLocation)) + .scoreMode(FunctionScoreMode.Sum) + .boostMode(FunctionBoostMode.Multiply) + .maxBoost(5.0))) + .minScore(minScore) + + .size(100) // Reduced from 500 + + .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) + + .source(src -> src + .filter(f -> f + .includes("benRegId", "beneficiaryID", "firstName", "lastName", + "genderID", "genderName", "dOB", "phoneNum", + "stateID", "districtID", "blockID", "villageID"))) + + , BeneficiariesESDTO.class); + + logger.info("ES returned {} hits in {}ms for query: '{}'", + response.hits().hits().size(), + response.took(), + query); + + if (response.hits().hits().isEmpty()) { + logger.info("No results in ES, using database fallback"); + return searchInDatabaseDirectly(query); + } + + List> results = response.hits().hits().stream() + .map(hit -> { + Map result = mapESResultToExpectedFormat(hit.source()); + if (result != null) { + result.put("_score", hit.score()); + } + return result; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + logger.info("Returning {} results", results.size()); + return results; + + } catch (Exception e) { + logger.error("ES search failed: {}", e.getMessage()); return searchInDatabaseDirectly(query); } + } - List> results = response.hits().hits().stream() - .map(hit -> { - Map result = mapESResultToExpectedFormat(hit.source()); - if (result != null) { - result.put("_score", hit.score()); - } - return result; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + /** + * Generate function scores for location-based ranking + */ + private List getFunctionScores(Map userLocation) { + if (userLocation == null) { + return List.of(); + } - logger.info("Returning {} results", results.size()); - return results; + List scores = new ArrayList<>(); - } catch (Exception e) { - logger.error("ES search failed: {}", e.getMessage()); - return searchInDatabaseDirectly(query); - } -} - -/** - * Generate function scores for location-based ranking - */ -private List getFunctionScores(Map userLocation) { - if (userLocation == null) { - return List.of(); - } + Integer userVillageId = userLocation.get("villageId"); + Integer userBlockId = userLocation.get("blockId"); - List scores = new ArrayList<>(); - - Integer userVillageId = userLocation.get("villageId"); - Integer userBlockId = userLocation.get("blockId"); - - // Village match - if (userVillageId != null) { - scores.add(FunctionScore.of(f -> f - .filter(ff -> ff.term(t -> t.field("villageID").value(userVillageId))) - .weight(2.0) - )); - } - - // Block match - if (userBlockId != null) { - scores.add(FunctionScore.of(f -> f - .filter(ff -> ff.term(t -> t.field("blockID").value(userBlockId))) - .weight(1.5) - )); - } - - return scores; -} - -/** - * Advanced search with filter context - */ -public List> advancedSearch( - String firstName, - String lastName, - Integer genderId, - Date dob, - Integer stateId, - Integer districtId, - Integer blockId, - Integer villageId, - String fatherName, - String spouseName, - String phoneNumber, - String beneficiaryId, - String healthId, - String aadharNo, - Integer userId) { - - try { - final Map userLocation = - (userId != null) ? getUserLocation(userId) : null; - - SearchResponse response = esClient.search(s -> s - .index(beneficiaryIndex) - .preference("_local") - .requestCache(true) - - .query(q -> q - .bool(b -> { - // Use FILTER context for exact matches (faster, cached) - if (genderId != null) { - b.filter(f -> f.term(t -> t.field("genderID").value(genderId))); - } - if (stateId != null) { - b.filter(f -> f.term(t -> t.field("stateID").value(stateId))); - } - if (districtId != null) { - b.filter(f -> f.term(t -> t.field("districtID").value(districtId))); - } - if (blockId != null) { - b.filter(f -> f.term(t -> t.field("blockID").value(blockId))); - } - if (villageId != null) { - b.filter(f -> f.term(t -> t.field("villageID").value(villageId))); - } - - // MUST context for scored searches - if (firstName != null && !firstName.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.term(t -> t.field("firstName.keyword").value(firstName).boost(5.0f))) - .should(s2 -> s2.match(mm -> mm.field("firstName").query(firstName).boost(2.0f))) - .minimumShouldMatch("1") - )); - } - - if (lastName != null && !lastName.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.term(t -> t.field("lastName.keyword").value(lastName).boost(5.0f))) - .should(s2 -> s2.match(mm -> mm.field("lastName").query(lastName).boost(2.0f))) - .minimumShouldMatch("1") - )); - } - - // Exact match IDs in filter context - if (beneficiaryId != null && !beneficiaryId.trim().isEmpty()) { - b.filter(f -> f.term(t -> t.field("beneficiaryID").value(beneficiaryId))); - } - if (healthId != null && !healthId.trim().isEmpty()) { - b.filter(f -> f.bool(bb -> bb - .should(s1 -> s1.term(t -> t.field("healthID").value(healthId))) - .should(s2 -> s2.term(t -> t.field("abhaID").value(healthId))) - .minimumShouldMatch("1") - )); - } - if (aadharNo != null && !aadharNo.trim().isEmpty()) { - b.filter(f -> f.term(t -> t.field("aadharNo").value(aadharNo))); - } - if (phoneNumber != null && !phoneNumber.trim().isEmpty()) { - b.must(m -> m.bool(bb -> bb - .should(s1 -> s1.term(t -> t.field("phoneNum").value(phoneNumber).boost(3.0f))) - .should(s2 -> s2.prefix(p -> p.field("phoneNum").value(phoneNumber).boost(2.0f))) - .minimumShouldMatch("1") - )); - } - - return b; - }) - ) - .size(100) - .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) - - , BeneficiariesESDTO.class); - - if (response.hits().hits().isEmpty()) { - return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, - stateId, districtId, blockId, villageId, fatherName, spouseName, - phoneNumber, beneficiaryId, healthId, aadharNo); + // Village match + if (userVillageId != null) { + scores.add(FunctionScore.of(f -> f + .filter(ff -> ff.term(t -> t.field("villageID").value(userVillageId))) + .weight(2.0))); } - return response.hits().hits().stream() - .map(hit -> mapESResultToExpectedFormat(hit.source())) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + // Block match + if (userBlockId != null) { + scores.add(FunctionScore.of(f -> f + .filter(ff -> ff.term(t -> t.field("blockID").value(userBlockId))) + .weight(1.5))); + } - } catch (Exception e) { - logger.error("ES advanced search failed: {}", e.getMessage()); - return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, - stateId, districtId, blockId, villageId, fatherName, spouseName, - phoneNumber, beneficiaryId, healthId, aadharNo); + return scores; } -} - -/** - * Database fallback for advanced search - */ -private List> searchInDatabaseForAdvanced( - String firstName, String lastName, Integer genderId, Date dob, - Integer stateId, Integer districtId, Integer blockId, Integer villageId, - String fatherName, String spouseName, String phoneNumber, - String beneficiaryId, String healthId, String aadharNo) { - - try { - List results = benDetailRepo.advancedSearchBeneficiaries( - firstName, lastName, genderId, dob, stateId, districtId, - blockId, fatherName, spouseName, phoneNumber, - beneficiaryId - ); - - return results.stream() - .map(this::mapToExpectedFormat) - .collect(Collectors.toList()); - - } catch (Exception e) { - logger.error("Database advanced search failed: {}", e.getMessage(), e); - return Collections.emptyList(); + + /** + * Advanced search with filter context + */ + public List> advancedSearch( + String firstName, + String lastName, + Integer genderId, + Date dob, + Integer stateId, + Integer districtId, + Integer blockId, + Integer villageId, + String fatherName, + String spouseName, + String phoneNumber, + String beneficiaryId, + String healthId, + String aadharNo, + Integer userId) { + + try { + final Map userLocation = (userId != null) ? getUserLocation(userId) : null; + + SearchResponse response = esClient.search(s -> s + .index(beneficiaryIndex) + .preference("_local") + .requestCache(true) + + .query(q -> q + .bool(b -> { + // Use FILTER context for exact matches (faster, cached) + if (genderId != null) { + b.filter(f -> f.term(t -> t.field("genderID").value(genderId))); + } + if (stateId != null) { + b.filter(f -> f.term(t -> t.field("stateID").value(stateId))); + } + if (districtId != null) { + b.filter(f -> f.term(t -> t.field("districtID").value(districtId))); + } + if (blockId != null) { + b.filter(f -> f.term(t -> t.field("blockID").value(blockId))); + } + if (villageId != null) { + b.filter(f -> f.term(t -> t.field("villageID").value(villageId))); + } + + // MUST context for scored searches + if (firstName != null && !firstName.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1.term( + t -> t.field("firstName.keyword").value(firstName).boost(5.0f))) + .should(s2 -> s2 + .match(mm -> mm.field("firstName").query(firstName).boost(2.0f))) + .minimumShouldMatch("1"))); + } + + if (lastName != null && !lastName.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1 + .term(t -> t.field("lastName.keyword").value(lastName).boost(5.0f))) + .should(s2 -> s2 + .match(mm -> mm.field("lastName").query(lastName).boost(2.0f))) + .minimumShouldMatch("1"))); + } + + // Exact match IDs in filter context + if (beneficiaryId != null && !beneficiaryId.trim().isEmpty()) { + b.filter(f -> f.term(t -> t.field("beneficiaryID").value(beneficiaryId))); + } + if (healthId != null && !healthId.trim().isEmpty()) { + b.filter(f -> f.bool(bb -> bb + .should(s1 -> s1.term(t -> t.field("healthID").value(healthId))) + .should(s2 -> s2.term(t -> t.field("abhaID").value(healthId))) + .minimumShouldMatch("1"))); + } + if (aadharNo != null && !aadharNo.trim().isEmpty()) { + b.filter(f -> f.term(t -> t.field("aadharNo").value(aadharNo))); + } + if (phoneNumber != null && !phoneNumber.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1 + .term(t -> t.field("phoneNum").value(phoneNumber).boost(3.0f))) + .should(s2 -> s2 + .prefix(p -> p.field("phoneNum").value(phoneNumber).boost(2.0f))) + .minimumShouldMatch("1"))); + } + + return b; + })) + .size(100) + .sort(so -> so.score(sc -> sc.order(SortOrder.Desc))) + + , BeneficiariesESDTO.class); + + if (response.hits().hits().isEmpty()) { + return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, + stateId, districtId, blockId, villageId, fatherName, spouseName, + phoneNumber, beneficiaryId, healthId, aadharNo); + } + + return response.hits().hits().stream() + .map(hit -> mapESResultToExpectedFormat(hit.source())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + } catch (Exception e) { + logger.error("ES advanced search failed: {}", e.getMessage()); + return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, + stateId, districtId, blockId, villageId, fatherName, spouseName, + phoneNumber, beneficiaryId, healthId, aadharNo); + } } -} + /** + * Database fallback for advanced search + */ + private List> searchInDatabaseForAdvanced( + String firstName, String lastName, Integer genderId, Date dob, + Integer stateId, Integer districtId, Integer blockId, Integer villageId, + String fatherName, String spouseName, String phoneNumber, + String beneficiaryId, String healthId, String aadharNo) { + + try { + List results = benDetailRepo.advancedSearchBeneficiaries( + firstName, lastName, genderId, dob, stateId, districtId, + blockId, fatherName, spouseName, phoneNumber, + beneficiaryId); + + return results.stream() + .map(this::mapToExpectedFormat) + .collect(Collectors.toList()); + + } catch (Exception e) { + logger.error("Database advanced search failed: {}", e.getMessage(), e); + return Collections.emptyList(); + } + } /** * Overloaded method without userId (backward compatibility) @@ -383,7 +375,7 @@ private List> searchInDatabaseForAdvanced( public List> universalSearch(String query) { return universalSearch(query, null); } - + /** * Get user location from database */ @@ -404,73 +396,73 @@ private Map getUserLocation(Integer userId) { } return null; } - + /** * Rank results by location match priority * Priority: Village > Block > District > State > No Match */ - private List> rankByLocation(List> results, - Map userLocation) { + private List> rankByLocation(List> results, + Map userLocation) { Integer userBlockId = userLocation.get("blockId"); Integer userVillageId = userLocation.get("villageId"); - + logger.info("Ranking by location - User Block: {}, Village: {}", userBlockId, userVillageId); - + return results.stream() - .sorted((r1, r2) -> { - int score1 = calculateLocationScore(r1, userBlockId, userVillageId); - int score2 = calculateLocationScore(r2, userBlockId, userVillageId); - return Integer.compare(score2, score1); - }) - .collect(Collectors.toList()); + .sorted((r1, r2) -> { + int score1 = calculateLocationScore(r1, userBlockId, userVillageId); + int score2 = calculateLocationScore(r2, userBlockId, userVillageId); + return Integer.compare(score2, score1); + }) + .collect(Collectors.toList()); } - + /** * Calculate location match score * Higher score = better match */ - private int calculateLocationScore(Map beneficiary, - Integer userBlockId, - Integer userVillageId) { + private int calculateLocationScore(Map beneficiary, + Integer userBlockId, + Integer userVillageId) { int score = 0; - + try { Map demographics = (Map) beneficiary.get("i_bendemographics"); if (demographics == null) { return score; } - + Integer currBlockId = getIntegerFromMap(demographics, "blockID"); Integer currVillageId = getIntegerFromMap(demographics, "m_districtblock", "blockID"); - + // Village match (highest priority) - score: 100 if (userVillageId != null && userVillageId.equals(currVillageId)) { score += 100; } - + // Block match - score: 50 if (userBlockId != null && userBlockId.equals(currBlockId)) { score += 50; } - + Integer permBlockId = getIntegerFromMap(beneficiary, "permBlockID"); Integer permVillageId = getIntegerFromMap(beneficiary, "permVillageID"); - + if (userVillageId != null && userVillageId.equals(permVillageId)) { - score += 75; + score += 75; } - + if (userBlockId != null && userBlockId.equals(permBlockId)) { - score += 25; + score += 25; } - + } catch (Exception e) { logger.error("Error calculating location score: {}", e.getMessage()); } - + return score; } - + /** * Helper to safely get Integer from nested maps */ @@ -485,7 +477,7 @@ private Integer getIntegerFromMap(Map map, String... keys) { } return value instanceof Integer ? (Integer) value : null; } - + /** * Map ES DTO directly to expected API format with COMPLETE data */ @@ -493,9 +485,9 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat if (esData == null) { return null; } - + Map result = new HashMap<>(); - + try { // Basic fields from ES result.put("beneficiaryRegID", esData.getBenRegId()); @@ -513,16 +505,16 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat result.put("createdDate", esData.getCreatedDate()); result.put("lastModDate", esData.getLastModDate()); result.put("benAccountID", esData.getBenAccountID()); - + result.put("healthID", esData.getHealthID()); result.put("abhaID", esData.getAbhaID()); result.put("familyID", esData.getFamilyID()); - + Map mGender = new HashMap<>(); mGender.put("genderID", esData.getGenderID()); mGender.put("genderName", esData.getGenderName()); result.put("m_gender", mGender); - + Map demographics = new HashMap<>(); demographics.put("beneficiaryRegID", esData.getBenRegId()); demographics.put("stateID", esData.getStateID()); @@ -539,36 +531,36 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat demographics.put("servicePointID", esData.getServicePointID()); demographics.put("servicePointName", esData.getServicePointName()); demographics.put("createdBy", esData.getCreatedBy()); - + Map mState = new HashMap<>(); mState.put("stateID", esData.getStateID()); mState.put("stateName", esData.getStateName()); mState.put("stateCode", null); mState.put("countryID", 1); demographics.put("m_state", mState); - + Map mDistrict = new HashMap<>(); mDistrict.put("districtID", esData.getDistrictID()); mDistrict.put("districtName", esData.getDistrictName()); mDistrict.put("stateID", esData.getStateID()); demographics.put("m_district", mDistrict); - + Map mBlock = new HashMap<>(); mBlock.put("blockID", esData.getBlockID()); mBlock.put("blockName", esData.getBlockName()); mBlock.put("districtID", esData.getDistrictID()); mBlock.put("stateID", esData.getStateID()); demographics.put("m_districtblock", mBlock); - + Map mBranch = new HashMap<>(); mBranch.put("districtBranchID", null); mBranch.put("blockID", esData.getBlockID()); mBranch.put("villageName", esData.getVillageName()); mBranch.put("pinCode", esData.getPinCode()); demographics.put("m_districtbranchmapping", mBranch); - + result.put("i_bendemographics", demographics); - + List> benPhoneMaps = new ArrayList<>(); if (esData.getPhoneNum() != null && !esData.getPhoneNum().isEmpty()) { Map phoneMap = new HashMap<>(); @@ -577,16 +569,16 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat phoneMap.put("parentBenRegID", esData.getBenRegId()); phoneMap.put("benRelationshipID", 1); phoneMap.put("phoneNo", esData.getPhoneNum()); - + Map relationType = new HashMap<>(); relationType.put("benRelationshipID", 1); relationType.put("benRelationshipType", "Self"); phoneMap.put("benRelationshipType", relationType); - + benPhoneMaps.add(phoneMap); } result.put("benPhoneMaps", benPhoneMaps); - + result.put("isConsent", false); result.put("m_title", new HashMap<>()); result.put("maritalStatus", new HashMap<>()); @@ -603,38 +595,38 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat result.put("emergencyRegistration", false); result.put("passToNurse", false); result.put("beneficiaryIdentities", new ArrayList<>()); - + } catch (Exception e) { logger.error("Error mapping ES result: {}", e.getMessage(), e); return null; } - + return result; } - + /** * Direct database search as fallback */ private List> searchInDatabaseDirectly(String query) { try { List results = benDetailRepo.searchBeneficiaries(query); - + return results.stream() - .map(this::mapToExpectedFormat) - .collect(Collectors.toList()); - + .map(this::mapToExpectedFormat) + .collect(Collectors.toList()); + } catch (Exception e) { logger.error("Database search failed: {}", e.getMessage(), e); return Collections.emptyList(); } } - + /** * Map database result to expected API format */ private Map mapToExpectedFormat(Object[] row) { Map result = new HashMap<>(); - + try { Long beneficiaryRegID = getLong(row[0]); String beneficiaryID = getString(row[1]); @@ -651,7 +643,7 @@ private Map mapToExpectedFormat(Object[] row) { Date createdDate = getDate(row[12]); Long lastModDate = getLong(row[13]); Long benAccountID = getLong(row[14]); - + Integer stateID = getInteger(row[15]); String stateName = getString(row[16]); Integer districtID = getInteger(row[17]); @@ -663,7 +655,7 @@ private Map mapToExpectedFormat(Object[] row) { String servicePointName = getString(row[23]); Integer parkingPlaceID = getInteger(row[24]); String phoneNum = getString(row[25]); - + result.put("beneficiaryRegID", beneficiaryRegID); result.put("beneficiaryID", beneficiaryID); result.put("firstName", firstName); @@ -679,12 +671,12 @@ private Map mapToExpectedFormat(Object[] row) { result.put("createdDate", createdDate); result.put("lastModDate", lastModDate); result.put("benAccountID", benAccountID); - + Map mGender = new HashMap<>(); mGender.put("genderID", genderID); mGender.put("genderName", genderName); result.put("m_gender", mGender); - + Map demographics = new HashMap<>(); demographics.put("beneficiaryRegID", beneficiaryRegID); demographics.put("stateID", stateID); @@ -699,39 +691,39 @@ private Map mapToExpectedFormat(Object[] row) { demographics.put("servicePointID", servicePointID); demographics.put("servicePointName", servicePointName); demographics.put("createdBy", createdBy); - + Map mState = new HashMap<>(); mState.put("stateID", stateID); mState.put("stateName", stateName); mState.put("stateCode", null); mState.put("countryID", 1); demographics.put("m_state", mState); - + Map mDistrict = new HashMap<>(); mDistrict.put("districtID", districtID); mDistrict.put("districtName", districtName); mDistrict.put("stateID", stateID); demographics.put("m_district", mDistrict); - + Map mBlock = new HashMap<>(); mBlock.put("blockID", blockID); mBlock.put("blockName", blockName); mBlock.put("districtID", districtID); mBlock.put("stateID", stateID); demographics.put("m_districtblock", mBlock); - + Map mBranch = new HashMap<>(); mBranch.put("districtBranchID", null); mBranch.put("blockID", blockID); mBranch.put("villageName", null); mBranch.put("pinCode", pinCode); demographics.put("m_districtbranchmapping", mBranch); - + result.put("i_bendemographics", demographics); - + List> benPhoneMaps = fetchPhoneNumbers(beneficiaryRegID); result.put("benPhoneMaps", benPhoneMaps); - + result.put("isConsent", false); result.put("m_title", new HashMap<>()); result.put("maritalStatus", new HashMap<>()); @@ -748,28 +740,28 @@ private Map mapToExpectedFormat(Object[] row) { result.put("emergencyRegistration", false); result.put("passToNurse", false); result.put("beneficiaryIdentities", new ArrayList<>()); - + } catch (Exception e) { logger.error("Error mapping result: {}", e.getMessage(), e); } - + return result; } - + /** * Fetch phone numbers for a beneficiary */ private List> fetchPhoneNumbers(Long beneficiaryRegID) { List> phoneList = new ArrayList<>(); - + try { List phones = benDetailRepo.findPhoneNumbersByBeneficiaryId(beneficiaryRegID); - + int mapId = 1; for (Object[] phone : phones) { String phoneNo = getString(phone[0]); String phoneType = getString(phone[1]); - + if (phoneNo != null && !phoneNo.isEmpty()) { Map phoneMap = new HashMap<>(); phoneMap.put("benPhMapID", (long) mapId++); @@ -777,33 +769,38 @@ private List> fetchPhoneNumbers(Long beneficiaryRegID) { phoneMap.put("parentBenRegID", beneficiaryRegID); phoneMap.put("benRelationshipID", 1); phoneMap.put("phoneNo", phoneNo); - + Map relationType = new HashMap<>(); relationType.put("benRelationshipID", 1); relationType.put("benRelationshipType", phoneType != null ? phoneType : "Self"); phoneMap.put("benRelationshipType", relationType); - + phoneList.add(phoneMap); } } } catch (Exception e) { logger.error("Error fetching phone numbers: {}", e.getMessage(), e); } - + return phoneList; } - + // Helper methods private String getString(Object value) { - if (value == null) return null; + if (value == null) + return null; return value.toString(); } - + private Long getLong(Object value) { - if (value == null) return null; - if (value instanceof Long) return (Long) value; - if (value instanceof Integer) return ((Integer) value).longValue(); - if (value instanceof BigDecimal) return ((BigDecimal) value).longValue(); + if (value == null) + return null; + if (value instanceof Long) + return (Long) value; + if (value instanceof Integer) + return ((Integer) value).longValue(); + if (value instanceof BigDecimal) + return ((BigDecimal) value).longValue(); if (value instanceof String) { try { return Long.parseLong((String) value); @@ -813,12 +810,16 @@ private Long getLong(Object value) { } return null; } - + private Integer getInteger(Object value) { - if (value == null) return null; - if (value instanceof Integer) return (Integer) value; - if (value instanceof Long) return ((Long) value).intValue(); - if (value instanceof BigDecimal) return ((BigDecimal) value).intValue(); + if (value == null) + return null; + if (value instanceof Integer) + return (Integer) value; + if (value instanceof Long) + return ((Long) value).intValue(); + if (value instanceof BigDecimal) + return ((BigDecimal) value).intValue(); if (value instanceof String) { try { return Integer.parseInt((String) value); @@ -828,12 +829,16 @@ private Integer getInteger(Object value) { } return null; } - + private Date getDate(Object value) { - if (value == null) return null; - if (value instanceof Date) return (Date) value; - if (value instanceof Timestamp) return new Date(((Timestamp) value).getTime()); - if (value instanceof java.sql.Date) return new Date(((java.sql.Date) value).getTime()); + if (value == null) + return null; + if (value instanceof Date) + return (Date) value; + if (value instanceof Timestamp) + return new Date(((Timestamp) value).getTime()); + if (value instanceof java.sql.Date) + return new Date(((java.sql.Date) value).getTime()); return null; } } \ No newline at end of file From 08656a5060f0124eb11bb733b1c4d02a34e5b9eb Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 20 Jan 2026 09:45:26 +0530 Subject: [PATCH 03/14] fix: abha / health id issue --- .../identity/repo/V_BenAdvanceSearchRepo.java | 5 + .../elasticsearch/BeneficiaryDataService.java | 53 +++- .../BeneficiaryDocumentDataService.java | 255 +++++++++++++++--- .../ElasticsearchIndexingService.java | 121 +++++++-- .../elasticsearch/ElasticsearchService.java | 18 +- .../ElasticsearchSyncService.java | 15 +- 6 files changed, 405 insertions(+), 62 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/repo/V_BenAdvanceSearchRepo.java b/src/main/java/com/iemr/common/identity/repo/V_BenAdvanceSearchRepo.java index 0be61bb..a7fd5f2 100644 --- a/src/main/java/com/iemr/common/identity/repo/V_BenAdvanceSearchRepo.java +++ b/src/main/java/com/iemr/common/identity/repo/V_BenAdvanceSearchRepo.java @@ -46,4 +46,9 @@ public interface V_BenAdvanceSearchRepo extends CrudRepository getBenRegIDByHealthIDNoAbhaIdNo(@Param("healthIDNo") String healthIDNo); + //Batch fetch ABHA details for multiple beneficiaries + @Query(nativeQuery = true, value = "SELECT BeneficiaryRegID, HealthID, HealthIDNumber, AuthenticationMode, CreatedDate" + + " FROM db_iemr.m_benhealthidmapping WHERE BeneficiaryRegID IN :benRegIDs ORDER BY BeneficiaryRegID, CreatedDate DESC") + List getBenAbhaDetailsByBenRegIDs(@Param("benRegIDs") List benRegIDs); + } diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java index 226ff7a..88854d4 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java @@ -1,6 +1,7 @@ package com.iemr.common.identity.service.elasticsearch; import java.math.BigInteger; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,9 +9,11 @@ import org.springframework.stereotype.Service; import com.iemr.common.identity.domain.MBeneficiarymapping; +import com.iemr.common.identity.dto.AbhaAddressDTO; import com.iemr.common.identity.dto.BenDetailDTO; import com.iemr.common.identity.dto.BeneficiariesDTO; import com.iemr.common.identity.repo.BenMappingRepo; +import com.iemr.common.identity.repo.V_BenAdvanceSearchRepo; /** * Service to fetch beneficiary data directly from database @@ -24,6 +27,10 @@ public class BeneficiaryDataService { @Autowired private BenMappingRepo mappingRepo; + @Autowired + private V_BenAdvanceSearchRepo v_BenAdvanceSearchRepo; + + /** * Fetch beneficiary data directly from database by benRegId * This bypasses any Elasticsearch caching to get fresh database data @@ -108,7 +115,51 @@ private BeneficiariesDTO convertToDTO(MBeneficiarymapping mapping) { dto.setBeneficiaryDetails(detailDTO); } - + try { + logger.info("Fetching ABHA details for benRegId={}", dto.getBenRegId()); + if (dto.getBenRegId() != null) { + List abhaList = v_BenAdvanceSearchRepo.getBenAbhaDetailsByBenRegID(dto.getBenRegId()); + + if (abhaList != null && !abhaList.isEmpty()) { + List abhaDTOList = new java.util.ArrayList<>(); + + for (Object[] objArr : abhaList) { + AbhaAddressDTO abhaDTO = new AbhaAddressDTO(); + abhaDTO.setBeneficiaryRegID(dto.getBenRegId()); + + // objArr[1] -> HealthID (ABHA Address) + if (objArr[1] != null) { + abhaDTO.setHealthID(objArr[1].toString()); + } + + // objArr[2] -> HealthIDNumber (ABHA Number) + if (objArr[2] != null) { + abhaDTO.setHealthIDNumber(objArr[2].toString()); + } + + // objArr[3] -> Authentication Mode + if (objArr[3] != null) { + abhaDTO.setAuthenticationMode(objArr[3].toString()); + } + + // objArr[4] -> Created Date + if (objArr[4] != null) { + abhaDTO.setCreatedDate((java.sql.Timestamp) objArr[4]); + } + + abhaDTOList.add(abhaDTO); + } + + dto.setAbhaDetails(abhaDTOList); + logger.info("ABHA details fetched: count={}", abhaDTOList.size()); + } + } +} catch (Exception e) { + logger.error("Error while fetching ABHA details for benRegId={}", + dto.getBenRegId(), e); +} + + logger.debug("Successfully converted mapping to DTO: benRegId={}", mapping.getBenRegId()); } catch (Exception e) { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java index e6aa965..e64ddc6 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java @@ -2,16 +2,20 @@ import java.math.BigInteger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import com.iemr.common.identity.data.elasticsearch.BeneficiaryDocument; import com.iemr.common.identity.repo.BenMappingRepo; +import com.iemr.common.identity.repo.V_BenAdvanceSearchRepo; /** * Optimized service to fetch complete beneficiary data in bulk @@ -25,6 +29,9 @@ public class BeneficiaryDocumentDataService { @Autowired private BenMappingRepo mappingRepo; + @Autowired + private V_BenAdvanceSearchRepo v_BenAdvanceSearchRepo; + /** * Fetch multiple beneficiaries with COMPLETE data in ONE query * This is the KEY method that replaces multiple individual queries @@ -37,17 +44,32 @@ public List getBeneficiariesBatch(List benRegId try { logger.debug("Fetching {} beneficiaries with complete data", benRegIds.size()); - + List results = mappingRepo.findCompleteDataByBenRegIds(benRegIds); - + logger.info("Fetched {} complete beneficiary records", results.size()); + // Batch fetch ABHA details for ALL beneficiaries at once + Map abhaMap = batchFetchAbhaData(benRegIds); + + logger.info("Fetched ABHA details for {} beneficiaries", abhaMap.size()); + List documents = new ArrayList<>(); for (Object[] row : results) { try { BeneficiaryDocument doc = mapRowToDocument(row); if (doc != null && doc.getBenId() != null) { + + AbhaData abhaData = abhaMap.get(doc.getBenRegId()); + if (abhaData != null) { + doc.setHealthID(abhaData.getHealthID()); + doc.setAbhaID(abhaData.getHealthIDNumber()); + logger.info("Enriched benRegId={} with healthID={}, abhaID={}", + doc.getBenRegId(), doc.getHealthID(), doc.getAbhaID()); + } else { + logger.debug("No ABHA details for benRegId={}", doc.getBenRegId()); + } documents.add(doc); } } catch (Exception e) { @@ -64,6 +86,86 @@ public List getBeneficiariesBatch(List benRegId } } + private Map batchFetchAbhaData(List benRegIds) { + try { + return batchFetchAbhaDetails(benRegIds); + } catch (Exception e) { + logger.warn("Error fetching ABHA details (will continue without ABHA data): {}", e.getMessage()); + return new HashMap<>(); // Return empty map to continue processing + } + } + + /** + * Batch fetch ABHA details for multiple beneficiaries + * Returns a map of benRegId -> AbhaData + */ + @Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = true, timeout = 30) + private Map batchFetchAbhaDetails(List benRegIds) { + Map abhaMap = new HashMap<>(); + + try { + if (benRegIds == null || benRegIds.isEmpty()) { + logger.info("No beneficiary IDs provided for ABHA fetch"); + return abhaMap; + } + logger.debug("Batch fetching ABHA details for {} beneficiaries", benRegIds.size()); + + List abhaRecords = null; + try { + abhaRecords = v_BenAdvanceSearchRepo.getBenAbhaDetailsByBenRegIDs(benRegIds); + } catch (Exception e) { + logger.warn("ABHA query returned error (likely no records): {}", e.getMessage()); + return abhaMap; // Return empty map - this is OK + } + + if (abhaRecords == null || abhaRecords.isEmpty()) { + logger.debug("No ABHA records found for this batch (this is normal)"); + return abhaMap; + } + + logger.debug("Retrieved {} ABHA records", abhaRecords.size()); + + for (Object[] record : abhaRecords) { + try { + // record[0] -> BeneficiaryRegID + // record[1] -> HealthID (ABHA Address) + // record[2] -> HealthIDNumber (ABHA Number) + // record[3] -> AuthenticationMode + // record[4] -> CreatedDate + + Long benRegId = null; + if (record[0] instanceof BigInteger) { + benRegId = ((BigInteger) record[0]).longValue(); + } else if (record[0] instanceof Long) { + benRegId = (Long) record[0]; + } else if (record[0] instanceof Integer) { + benRegId = ((Integer) record[0]).longValue(); + } + + if (benRegId != null && !abhaMap.containsKey(benRegId)) { + // Only store the first (most recent) record for each beneficiary + AbhaData abhaData = new AbhaData(); + abhaData.setHealthID(record[1] != null ? record[1].toString() : null); + abhaData.setHealthIDNumber(record[2] != null ? record[2].toString() : null); + abhaData.setAuthenticationMode(record[3] != null ? record[3].toString() : null); + + abhaMap.put(benRegId, abhaData); + } + } catch (Exception e) { + logger.error("Error processing ABHA record: {}", e.getMessage()); + // Continue processing other records + } + } + + logger.debug("Processed {} unique ABHA records into map", abhaMap.size()); + + } catch (Exception e) { + logger.error("Error batch fetching ABHA details: {}", e.getMessage(), e); + } + + return abhaMap; + } + /** * Single beneficiary fetch (for backward compatibility) */ @@ -71,52 +173,87 @@ public List getBeneficiariesBatch(List benRegId public BeneficiaryDocument getBeneficiaryFromDatabase(BigInteger benRegId) { List ids = new ArrayList<>(); ids.add(benRegId); - + List results = getBeneficiariesBatch(ids); return results.isEmpty() ? null : results.get(0); } - + + /** + * Enrich document with ABHA details from m_benhealthidmapping table + */ + private void enrichWithAbhaDetails(BeneficiaryDocument doc) { + try { + if (doc.getBenRegId() == null) { + return; + } + + BigInteger benRegId = BigInteger.valueOf(doc.getBenRegId()); + List abhaList = v_BenAdvanceSearchRepo.getBenAbhaDetailsByBenRegID(benRegId); + + if (abhaList != null && !abhaList.isEmpty()) { + // Get the first (most recent) ABHA record + Object[] abhaRecord = abhaList.get(0); + + // objArr[1] -> HealthID (ABHA Address) + if (abhaRecord[1] != null) { + doc.setHealthID(abhaRecord[1].toString()); + } + + // objArr[2] -> HealthIDNumber (ABHA Number) + if (abhaRecord[2] != null) { + doc.setAbhaID(abhaRecord[2].toString()); + } + + logger.info("ABHA details enriched for benRegId={}, healthID={}, abhaID={}", + benRegId, doc.getHealthID(), doc.getAbhaID()); + } + } catch (Exception e) { + logger.error("Error enriching ABHA details for benRegId={}: {}", + doc.getBenRegId(), e.getMessage()); + } + } + /** * Map database row to BeneficiaryDocument for ES * Matches the query column order from BenMappingRepo */ private BeneficiaryDocument mapRowToDocument(Object[] row) { BeneficiaryDocument doc = new BeneficiaryDocument(); - + try { int idx = 0; - + // Basic IDs (0-1) Long benRegId = getLong(row[idx++]); doc.setBenRegId(benRegId); - String beneficiaryID = getString(row[idx++]); - if (beneficiaryID != null && !beneficiaryID.isEmpty()) { + String beneficiaryID = getString(row[idx++]); + if (beneficiaryID != null && !beneficiaryID.isEmpty()) { doc.setBenId(beneficiaryID); - } + } doc.setBeneficiaryID(beneficiaryID); - + doc.setFirstName(getString(row[idx++])); doc.setLastName(getString(row[idx++])); doc.setGenderID(getInteger(row[idx++])); doc.setGenderName(getString(row[idx++])); - doc.setGender(doc.getGenderName()); + doc.setGender(doc.getGenderName()); doc.setDOB(getDate(row[idx++])); doc.setAge(getInteger(row[idx++])); doc.setFatherName(getString(row[idx++])); doc.setSpouseName(getString(row[idx++])); doc.setIsHIVPos(getString(row[idx++])); - + doc.setCreatedBy(getString(row[idx++])); doc.setCreatedDate(getDate(row[idx++])); doc.setLastModDate(getLong(row[idx++])); doc.setBenAccountID(getLong(row[idx++])); - + doc.setPhoneNum(getString(row[idx++])); - - doc.setHealthID(getString(row[idx++])); - doc.setAbhaID(getString(row[idx++])); + + // doc.setHealthID(getString(row[idx++])); + // doc.setAbhaID(getString(row[idx++])); doc.setFamilyID(getString(row[idx++])); - + doc.setStateID(getInteger(row[idx++])); doc.setStateName(getString(row[idx++])); doc.setDistrictID(getInteger(row[idx++])); @@ -129,7 +266,7 @@ private BeneficiaryDocument mapRowToDocument(Object[] row) { doc.setServicePointID(getInteger(row[idx++])); doc.setServicePointName(getString(row[idx++])); doc.setParkingPlaceID(getInteger(row[idx++])); - + doc.setPermStateID(getInteger(row[idx++])); doc.setPermStateName(getString(row[idx++])); doc.setPermDistrictID(getInteger(row[idx++])); @@ -138,54 +275,98 @@ private BeneficiaryDocument mapRowToDocument(Object[] row) { doc.setPermBlockName(getString(row[idx++])); doc.setPermVillageID(getInteger(row[idx++])); doc.setPermVillageName(getString(row[idx++])); - + // doc.setGovtIdentityNo(getString(row[idx++])); // String aadhar = getString(row[idx]); // doc.setAadharNo(aadhar != null ? aadhar : doc.getGovtIdentityNo()); - + } catch (Exception e) { logger.error("Error mapping row to document: {}", e.getMessage(), e); } - + return doc; } - + // Helper methods private String getString(Object value) { return value != null ? value.toString() : null; } - + private Long getLong(Object value) { - if (value == null) return null; - if (value instanceof Long) return (Long) value; - if (value instanceof Integer) return ((Integer) value).longValue(); - if (value instanceof BigInteger) return ((BigInteger) value).longValue(); + if (value == null) + return null; + if (value instanceof Long) + return (Long) value; + if (value instanceof Integer) + return ((Integer) value).longValue(); + if (value instanceof BigInteger) + return ((BigInteger) value).longValue(); try { return Long.parseLong(value.toString()); } catch (NumberFormatException e) { return null; } } - + private Integer getInteger(Object value) { - if (value == null) return null; - if (value instanceof Integer) return (Integer) value; - if (value instanceof Long) return ((Long) value).intValue(); - if (value instanceof BigInteger) return ((BigInteger) value).intValue(); + if (value == null) + return null; + if (value instanceof Integer) + return (Integer) value; + if (value instanceof Long) + return ((Long) value).intValue(); + if (value instanceof BigInteger) + return ((BigInteger) value).intValue(); try { return Integer.parseInt(value.toString()); } catch (NumberFormatException e) { return null; } } - + private java.util.Date getDate(Object value) { - if (value == null) return null; - if (value instanceof java.util.Date) return (java.util.Date) value; - if (value instanceof java.sql.Timestamp) + if (value == null) + return null; + if (value instanceof java.util.Date) + return (java.util.Date) value; + if (value instanceof java.sql.Timestamp) return new java.util.Date(((java.sql.Timestamp) value).getTime()); - if (value instanceof java.sql.Date) + if (value instanceof java.sql.Date) return new java.util.Date(((java.sql.Date) value).getTime()); return null; } + + /** + * Inner class to hold ABHA data + */ + private static class AbhaData { + private String healthID; + private String healthIDNumber; + private String authenticationMode; + + public String getHealthID() { + return healthID; + } + + public void setHealthID(String healthID) { + this.healthID = healthID; + } + + public String getHealthIDNumber() { + return healthIDNumber; + } + + public void setHealthIDNumber(String healthIDNumber) { + this.healthIDNumber = healthIDNumber; + } + + public String getAuthenticationMode() { + return authenticationMode; + } + + public void setAuthenticationMode(String authenticationMode) { + this.authenticationMode = authenticationMode; + } + } + } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java index 458dedf..9cac76c 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java @@ -2,8 +2,6 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.mapping.*; -import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; -import co.elastic.clients.elasticsearch.indices.IndexSettings; import co.elastic.clients.elasticsearch.indices.TranslogDurability; import org.slf4j.Logger; @@ -33,7 +31,7 @@ public class ElasticsearchIndexingService { */ public void createIndexWithMapping() throws Exception { - logger.info("Creating index with mapping: {}", beneficiaryIndex); + logger.info("Creating index optimized for bulk indexing: {}", beneficiaryIndex); // Delete existing index if it exists if (esClient.indices().exists(e -> e.index(beneficiaryIndex)).value()) { @@ -42,36 +40,59 @@ public void createIndexWithMapping() throws Exception { } IndexSettings settings = IndexSettings.of(s -> s - .refreshInterval(t -> t.time("30s")) + .refreshInterval(t -> t.time("-1")) // -1 = disable refresh completely - .numberOfShards("3") + // Use 1 shard for datasets < 50GB (yours is ~784K records) + .numberOfShards("1") - .numberOfReplicas("1") + // No replicas during initial indexing + .numberOfReplicas("0") + // Disable query cache during indexing .queries(q -> q - .cache(c -> c.enabled(true)) + .cache(c -> c.enabled(false)) ) .maxResultWindow(10000) .translog(t -> t - .durability(TranslogDurability.Async) - .syncInterval(ts -> ts.time("30s")) + .durability(TranslogDurability.Async) + .syncInterval(ts -> ts.time("120s")) // Longer interval for bulk ops + .flushThresholdSize(fb -> fb.bytes("1gb")) // Larger flush threshold + ) + + // Disable merge throttling during bulk indexing + .merge(m -> m + .scheduler(ms -> ms + .maxThreadCount(1) + .maxMergeCount(6) + ) + ) + + // Optimize for write performance + .indexing(i -> i + .slowlog(sl -> sl + .threshold(t -> t + .index(idx -> idx + .warn(w -> w.time("10s")) + .info(inf -> inf.time("5s")) + ) + ) + ) ) ); TypeMapping mapping = TypeMapping.of(tm -> tm .properties("benId", Property.of(p -> p.keyword(k -> k))) .properties("benRegId", Property.of(p -> p.long_(l -> l))) - .properties("beneficiaryID", Property.of(p -> p.keyword(k -> k))) .properties("firstName", Property.of(p -> p.text(t -> t - .analyzer("standard") + .analyzer("standard") .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) .fields("prefix", Property.of(fp -> fp.text(txt -> txt .analyzer("standard") - .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) // Fast prefix search + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) ))) ))) @@ -116,6 +137,7 @@ public void createIndexWithMapping() throws Exception { .properties("abhaID", Property.of(p -> p.keyword(k -> k))) .properties("familyID", Property.of(p -> p.keyword(k -> k))) + // Geographic fields .properties("stateID", Property.of(p -> p.integer(i -> i))) .properties("stateName", Property.of(p -> p.keyword(k -> k))) .properties("districtID", Property.of(p -> p.integer(i -> i))) @@ -129,6 +151,7 @@ public void createIndexWithMapping() throws Exception { .properties("servicePointName", Property.of(p -> p.keyword(k -> k))) .properties("parkingPlaceID", Property.of(p -> p.integer(i -> i))) + // Permanent address fields .properties("permStateID", Property.of(p -> p.integer(i -> i))) .properties("permStateName", Property.of(p -> p.keyword(k -> k))) .properties("permDistrictID", Property.of(p -> p.integer(i -> i))) @@ -138,6 +161,7 @@ public void createIndexWithMapping() throws Exception { .properties("permVillageID", Property.of(p -> p.integer(i -> i))) .properties("permVillageName", Property.of(p -> p.keyword(k -> k))) + // Identity fields .properties("aadharNo", Property.of(p -> p.keyword(k -> k))) .properties("govtIdentityNo", Property.of(p -> p.keyword(k -> k))) ); @@ -152,51 +176,104 @@ public void createIndexWithMapping() throws Exception { } /** - * Reset refresh interval after bulk indexing completes - * Call this after syncAllBeneficiaries() finishes + Optimize for search after bulk indexing completes */ public void optimizeForSearch() throws Exception { logger.info("Optimizing index for search performance..."); + // Step 1: Force refresh to make all documents searchable + logger.info("Forcing refresh to make documents visible..."); + esClient.indices().refresh(r -> r.index(beneficiaryIndex)); + + // Step 2: Update settings for search optimization + logger.info("Updating index settings for production search..."); esClient.indices().putSettings(s -> s .index(beneficiaryIndex) .settings(is -> is .refreshInterval(t -> t.time("1s")) - .translog(t -> t.durability(TranslogDurability.Request)) + .numberOfReplicas("1") + .translog(t -> t + .durability(TranslogDurability.Request) + .syncInterval(ts -> ts.time("5s")) + ) + .queries(q -> q + .cache(c -> c.enabled(true)) + ) ) ); + // Step 3: Force merge to optimize segment count + logger.info("Force merging segments for optimal read performance..."); esClient.indices().forcemerge(f -> f .index(beneficiaryIndex) - .maxNumSegments(1L) // Optimal for read-heavy workloads + .maxNumSegments(1L) // Single segment per shard for best performance + .flush(true) ); + logger.info("Index optimization completed"); } /** - * Index all beneficiaries - delegates to existing sync service + * Full indexing workflow with progress tracking */ - public Map indexAllBeneficiaries() { - logger.info("Starting full indexing via sync service..."); + public Map indexAllBeneficiaries() { + logger.info("STARTING FULL BENEFICIARY INDEXING"); + + long startTime = System.currentTimeMillis(); try { + // Execute bulk indexing + logger.info("PHASE 1: Bulk indexing beneficiaries..."); ElasticsearchSyncService.SyncResult result = syncService.syncAllBeneficiaries(); - // After indexing completes, optimize for search + long indexingTime = System.currentTimeMillis() - startTime; + logger.info("Bulk indexing completed in {} seconds", indexingTime / 1000); + logger.info("Success: {}, Failed: {}", result.getSuccessCount(), result.getFailureCount()); + + // Optimize for search + logger.info("PHASE 2: Optimizing for search..."); + long optimizeStart = System.currentTimeMillis(); optimizeForSearch(); + long optimizeTime = System.currentTimeMillis() - optimizeStart; + logger.info("Optimization completed in {} seconds", optimizeTime / 1000); - Map response = new HashMap<>(); + // Prepare response + Map response = new HashMap<>(); response.put("success", result.getSuccessCount()); response.put("failed", result.getFailureCount()); + response.put("indexingTimeSeconds", indexingTime / 1000); + response.put("optimizationTimeSeconds", optimizeTime / 1000); + response.put("totalTimeSeconds", (System.currentTimeMillis() - startTime) / 1000); + + logger.info("INDEXING COMPLETE - Total time: {} seconds", + (System.currentTimeMillis() - startTime) / 1000); return response; } catch (Exception e) { logger.error("Error during indexing", e); - Map response = new HashMap<>(); + Map response = new HashMap<>(); response.put("success", 0); response.put("failed", 0); + response.put("error", e.getMessage()); return response; } } + + /** + * Check index health and statistics + */ + public Map getIndexStats() throws Exception { + var stats = esClient.indices().stats(s -> s.index(beneficiaryIndex)); + var settings = esClient.indices().getSettings(g -> g.index(beneficiaryIndex)); + + Map info = new HashMap<>(); + info.put("documentCount", stats.indices().get(beneficiaryIndex).primaries().docs().count()); + info.put("sizeInBytes", stats.indices().get(beneficiaryIndex).primaries().store().sizeInBytes()); + info.put("refreshInterval", settings.get(beneficiaryIndex).settings().index().refreshInterval().time()); + info.put("numberOfShards", settings.get(beneficiaryIndex).settings().index().numberOfShards()); + info.put("numberOfReplicas", settings.get(beneficiaryIndex).settings().index().numberOfReplicas()); + + return info; + } } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index 430392c..b0836cf 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -166,7 +166,13 @@ public List> universalSearch(String query, Integer userId) { .filter(f -> f .includes("benRegId", "beneficiaryID", "firstName", "lastName", "genderID", "genderName", "dOB", "phoneNum", - "stateID", "districtID", "blockID", "villageID"))) + "stateID", "districtID", "blockID", "villageID", "healthID", "abhaID", "familyID", + "fatherName", "spouseName", "age", "createdBy", "createdDate", + "lastModDate", "benAccountID", "districtName", "blockName", + "villageName", "pinCode", "servicePointID", "servicePointName", + "parkingPlaceID", "permStateID", "permStateName", "permDistrictID", + "permDistrictName", "permBlockID", "permBlockName", "permVillageID", + "permVillageName"))) , BeneficiariesESDTO.class); @@ -175,6 +181,14 @@ public List> universalSearch(String query, Integer userId) { response.took(), query); + if (!response.hits().hits().isEmpty()) { + BeneficiariesESDTO firstResult = response.hits().hits().get(0).source(); + logger.info("First result - benRegId: {}, healthID: {}, abhaID: {}", + firstResult.getBenRegId(), + firstResult.getHealthID(), + firstResult.getAbhaID()); +} + if (response.hits().hits().isEmpty()) { logger.info("No results in ES, using database fallback"); return searchInDatabaseDirectly(query); @@ -486,6 +500,8 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat return null; } + logger.info("ESDATA="+esData.getAbhaID()); + Map result = new HashMap<>(); try { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index 8f40e37..140ab6e 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -17,10 +17,10 @@ import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import com.iemr.common.identity.data.elasticsearch.BeneficiaryDocument; +import com.iemr.common.identity.dto.AbhaAddressDTO; import com.iemr.common.identity.dto.BenDetailDTO; import com.iemr.common.identity.dto.BeneficiariesDTO; import com.iemr.common.identity.repo.BenMappingRepo; -import com.iemr.common.identity.service.elasticsearch.BeneficiaryDataService; /** * Service to synchronize beneficiary data from database to Elasticsearch @@ -336,7 +336,20 @@ private BeneficiaryDocument convertToDocument(BeneficiariesDTO dto) { doc.setAge(benDetails.getBeneficiaryAge()); doc.setGender(benDetails.getGender()); } + // ===== EXTRACT ABHA DETAILS ===== + if (dto.getAbhaDetails() != null && !dto.getAbhaDetails().isEmpty()) { + AbhaAddressDTO abhaDTO = dto.getAbhaDetails().get(0); + if (abhaDTO.getHealthID() != null) { + doc.setHealthID(abhaDTO.getHealthID()); + logger.debug("Set healthID={} for benRegId={}", abhaDTO.getHealthID(), doc.getBenRegId()); + } + + if (abhaDTO.getHealthIDNumber() != null) { + doc.setAbhaID(abhaDTO.getHealthIDNumber()); + logger.debug("Set abhaID={} for benRegId={}", abhaDTO.getHealthIDNumber(), doc.getBenRegId()); + } + } logger.debug("Successfully converted DTO to document: benId={}", doc.getBenId()); return doc; From 3c7d57add6bd025d4767f02e0da9ede5c310dda9 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 20 Jan 2026 11:18:10 +0530 Subject: [PATCH 04/14] fix: sync optimization --- .../config/ElasticsearchSyncConfig.java | 6 +- .../ElasticsearchIndexingService.java | 111 +++-- .../ElasticsearchSyncService.java | 378 ++++++++---------- src/main/resources/application.properties | 69 ++-- 4 files changed, 252 insertions(+), 312 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/config/ElasticsearchSyncConfig.java b/src/main/java/com/iemr/common/identity/config/ElasticsearchSyncConfig.java index bb72f91..ce6f5da 100644 --- a/src/main/java/com/iemr/common/identity/config/ElasticsearchSyncConfig.java +++ b/src/main/java/com/iemr/common/identity/config/ElasticsearchSyncConfig.java @@ -25,11 +25,13 @@ public Executor elasticsearchSyncExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); // Only 1-2 sync jobs should run at a time to avoid overwhelming DB/ES - executor.setCorePoolSize(5); - executor.setMaxPoolSize(10); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(4); executor.setQueueCapacity(100); executor.setThreadNamePrefix("es-sync-"); executor.setKeepAliveSeconds(60); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); // Handle rejected tasks executor.setRejectedExecutionHandler((r, executor1) -> { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java index 9cac76c..2b50c49 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java @@ -2,6 +2,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.mapping.*; +import co.elastic.clients.elasticsearch.indices.IndexSettings; import co.elastic.clients.elasticsearch.indices.TranslogDurability; import org.slf4j.Logger; @@ -28,7 +29,8 @@ public class ElasticsearchIndexingService { private String beneficiaryIndex; /** - + * Create index optimized for BULK INDEXING + * Settings will be updated for search after sync completes */ public void createIndexWithMapping() throws Exception { logger.info("Creating index optimized for bulk indexing: {}", beneficiaryIndex); @@ -39,13 +41,15 @@ public void createIndexWithMapping() throws Exception { esClient.indices().delete(d -> d.index(beneficiaryIndex)); } + // PHASE 1 SETTINGS: Optimized for BULK INDEXING (maximum write speed) IndexSettings settings = IndexSettings.of(s -> s - .refreshInterval(t -> t.time("-1")) // -1 = disable refresh completely + // CRITICAL: Disable refresh during bulk indexing + .refreshInterval(t -> t.time("-1")) // -1 = disable completely for max speed - // Use 1 shard for datasets < 50GB (yours is ~784K records) + // Use 1 shard for datasets < 50GB (optimal for 784K records) .numberOfShards("1") - // No replicas during initial indexing + // No replicas during initial indexing (add later for HA) .numberOfReplicas("0") // Disable query cache during indexing @@ -55,33 +59,14 @@ public void createIndexWithMapping() throws Exception { .maxResultWindow(10000) + // CRITICAL: Async translog for maximum speed .translog(t -> t .durability(TranslogDurability.Async) .syncInterval(ts -> ts.time("120s")) // Longer interval for bulk ops - .flushThresholdSize(fb -> fb.bytes("1gb")) // Larger flush threshold - ) - - // Disable merge throttling during bulk indexing - .merge(m -> m - .scheduler(ms -> ms - .maxThreadCount(1) - .maxMergeCount(6) - ) - ) - - // Optimize for write performance - .indexing(i -> i - .slowlog(sl -> sl - .threshold(t -> t - .index(idx -> idx - .warn(w -> w.time("10s")) - .info(inf -> inf.time("5s")) - ) - ) - ) ) ); + // Field mappings (supports fast search) TypeMapping mapping = TypeMapping.of(tm -> tm .properties("benId", Property.of(p -> p.keyword(k -> k))) .properties("benRegId", Property.of(p -> p.long_(l -> l))) @@ -172,96 +157,106 @@ public void createIndexWithMapping() throws Exception { .mappings(mapping) ); - logger.info("Index created successfully: {}", beneficiaryIndex); + logger.info("Index created with BULK INDEXING optimization"); + logger.info("Settings: refresh=disabled, replicas=0, async_translog, 1 shard"); } /** - Optimize for search after bulk indexing completes + * PHASE 2: Optimize for SEARCH after bulk indexing completes + * Call this AFTER indexAllBeneficiaries() finishes */ public void optimizeForSearch() throws Exception { - logger.info("Optimizing index for search performance..."); + logger.info("PHASE 2: Optimizing index for SEARCH performance"); // Step 1: Force refresh to make all documents searchable - logger.info("Forcing refresh to make documents visible..."); + logger.info("Step 1/3: Forcing refresh to make documents visible..."); esClient.indices().refresh(r -> r.index(beneficiaryIndex)); + logger.info("Documents are now searchable"); - // Step 2: Update settings for search optimization - logger.info("Updating index settings for production search..."); + // Step 2: Update settings for production search + logger.info("Step 2/3: Updating index settings for production..."); esClient.indices().putSettings(s -> s .index(beneficiaryIndex) .settings(is -> is - .refreshInterval(t -> t.time("1s")) - .numberOfReplicas("1") + .refreshInterval(t -> t.time("1s")) // Enable 1s refresh for near real-time search + .numberOfReplicas("1") // Add replica for high availability .translog(t -> t - .durability(TranslogDurability.Request) + .durability(TranslogDurability.Request) // Synchronous for data safety .syncInterval(ts -> ts.time("5s")) ) .queries(q -> q - .cache(c -> c.enabled(true)) + .cache(c -> c.enabled(true)) // Enable query cache for faster searches ) ) ); + logger.info("Settings applied: refresh=1s, replicas=1, query_cache=enabled"); - // Step 3: Force merge to optimize segment count - logger.info("Force merging segments for optimal read performance..."); + // Step 3: Force merge to optimize segments + logger.info("Step 3/3: Force merging segments for optimal read performance..."); + logger.info("This may take 5-15 minutes depending on data size..."); esClient.indices().forcemerge(f -> f .index(beneficiaryIndex) - .maxNumSegments(1L) // Single segment per shard for best performance + .maxNumSegments(1L) // Single segment per shard = fastest searches .flush(true) ); + logger.info("Segments merged to 1 per shard"); - logger.info("Index optimization completed"); + logger.info("INDEX OPTIMIZATION COMPLETE!"); + logger.info("Index is now ready for searches"); } /** - * Full indexing workflow with progress tracking + * COMPLETE WORKFLOW: Create index + Sync data + Optimize + * This is your existing endpoint, now with automatic optimization */ - public Map indexAllBeneficiaries() { - logger.info("STARTING FULL BENEFICIARY INDEXING"); + public Map indexAllBeneficiaries() { + logger.info("COMPLETE INDEXING WORKFLOW"); long startTime = System.currentTimeMillis(); try { - // Execute bulk indexing - logger.info("PHASE 1: Bulk indexing beneficiaries..."); + // Execute bulk indexing (now uses optimized batch queries) + logger.info("PHASE 1: Bulk indexing beneficiaries with batch queries..."); ElasticsearchSyncService.SyncResult result = syncService.syncAllBeneficiaries(); long indexingTime = System.currentTimeMillis() - startTime; - logger.info("Bulk indexing completed in {} seconds", indexingTime / 1000); + logger.info("Bulk indexing completed in {} seconds ({} minutes)", + indexingTime / 1000, indexingTime / 60000); logger.info("Success: {}, Failed: {}", result.getSuccessCount(), result.getFailureCount()); // Optimize for search + logger.info(""); logger.info("PHASE 2: Optimizing for search..."); long optimizeStart = System.currentTimeMillis(); optimizeForSearch(); long optimizeTime = System.currentTimeMillis() - optimizeStart; - logger.info("Optimization completed in {} seconds", optimizeTime / 1000); + logger.info("Optimization completed in {} seconds ({} minutes)", + optimizeTime / 1000, optimizeTime / 60000); + + long totalTime = System.currentTimeMillis() - startTime; - // Prepare response - Map response = new HashMap<>(); + logger.info("COMPLETE WORKFLOW FINISHED!"); + logger.info("Total time: {} seconds ({} minutes)", totalTime / 1000, totalTime / 60000); + logger.info("Indexing: {}m | Optimization: {}m", indexingTime / 60000, optimizeTime / 60000); + + // Return response in your existing format + Map response = new HashMap<>(); response.put("success", result.getSuccessCount()); response.put("failed", result.getFailureCount()); - response.put("indexingTimeSeconds", indexingTime / 1000); - response.put("optimizationTimeSeconds", optimizeTime / 1000); - response.put("totalTimeSeconds", (System.currentTimeMillis() - startTime) / 1000); - - logger.info("INDEXING COMPLETE - Total time: {} seconds", - (System.currentTimeMillis() - startTime) / 1000); return response; } catch (Exception e) { - logger.error("Error during indexing", e); - Map response = new HashMap<>(); + logger.error("Error during indexing workflow", e); + Map response = new HashMap<>(); response.put("success", 0); response.put("failed", 0); - response.put("error", e.getMessage()); return response; } } /** - * Check index health and statistics + * Get index statistics (unchanged) */ public Map getIndexStats() throws Exception { var stats = esClient.indices().stats(s -> s.index(beneficiaryIndex)); diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index 140ab6e..ab9ed32 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -17,10 +17,6 @@ import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import com.iemr.common.identity.data.elasticsearch.BeneficiaryDocument; -import com.iemr.common.identity.dto.AbhaAddressDTO; -import com.iemr.common.identity.dto.BenDetailDTO; -import com.iemr.common.identity.dto.BeneficiariesDTO; -import com.iemr.common.identity.repo.BenMappingRepo; /** * Service to synchronize beneficiary data from database to Elasticsearch @@ -29,36 +25,35 @@ public class ElasticsearchSyncService { private static final Logger logger = LoggerFactory.getLogger(ElasticsearchSyncService.class); - private static final int BATCH_SIZE = 100; // Reduced to 100 for better connection management - private static final int ES_BULK_SIZE = 50; // Reduced to 50 for better performance - private static final int PAUSE_AFTER_BATCHES = 5; // Pause after every 5 batches - @Autowired - private ElasticsearchClient esClient; + // OPTIMIZED BATCH SIZES for maximum performance + private static final int DB_FETCH_SIZE = 5000; // Fetch 5000 IDs at once + private static final int ES_BULK_SIZE = 2000; // Index 2000 documents per bulk request @Autowired - private BenMappingRepo mappingRepo; + private ElasticsearchClient esClient; @Autowired - private BeneficiaryDataService beneficiaryDataService; + private BeneficiaryTransactionHelper transactionalWrapper; @Autowired - private BeneficiaryTransactionHelper transactionalWrapper; + private BeneficiaryDocumentDataService documentDataService; // KEY: Batch service with ABHA @Value("${elasticsearch.index.beneficiary}") private String beneficiaryIndex; /** - * Sync all beneficiaries from database to Elasticsearch - * This should be run as a one-time operation or scheduled job + * Sync all beneficiaries using BATCH queries WITH ABHA + * This replaces individual queries with batch fetching (50-100x faster) */ public SyncResult syncAllBeneficiaries() { - logger.info("Starting full beneficiary sync to Elasticsearch..."); + logger.info("STARTING OPTIMIZED BATCH SYNC WITH ABHA"); SyncResult result = new SyncResult(); + long startTime = System.currentTimeMillis(); try { - // Get total count using transactional wrapper + // Get total count long totalCount = transactionalWrapper.countActiveBeneficiaries(); logger.info("Total beneficiaries to sync: {}", totalCount); @@ -68,104 +63,77 @@ public SyncResult syncAllBeneficiaries() { } AtomicInteger processedCount = new AtomicInteger(0); + AtomicInteger abhaEnrichedCount = new AtomicInteger(0); int offset = 0; - int batchCounter = 0; - List esBatch = new ArrayList<>(); + List esBatch = new ArrayList<>(ES_BULK_SIZE); - // Process in batches + // Process in large chunks for maximum speed while (offset < totalCount) { - logger.info("Fetching batch: offset={}, limit={}", offset, BATCH_SIZE); - - List batchIds = null; + long chunkStart = System.currentTimeMillis(); - try { - // Use transactional wrapper to get fresh connection for each batch - batchIds = transactionalWrapper.getBeneficiaryIdsBatch(offset, BATCH_SIZE); - } catch (Exception e) { - logger.error("Error fetching batch from database: {}", e.getMessage()); - // Wait and retry once - try { - Thread.sleep(2000); - batchIds = transactionalWrapper.getBeneficiaryIdsBatch(offset, BATCH_SIZE); - } catch (Exception e2) { - logger.error("Retry failed: {}", e2.getMessage()); - result.setError("Database connection error: " + e2.getMessage()); - break; - } - } + // STEP 1: Fetch IDs in batch + List batchIds = fetchBatchWithRetry(offset, DB_FETCH_SIZE); if (batchIds == null || batchIds.isEmpty()) { - logger.info("No more records to process. Breaking loop."); + logger.info("No more records to process."); break; } - logger.info("Processing {} beneficiaries in current batch", batchIds.size()); + logger.info("Fetched {} IDs at offset {}", batchIds.size(), offset); - for (Object[] benIdObj : batchIds) { - try { - - Object idObj = benIdObj[0]; - BigInteger benRegId; + // STEP 2: Convert IDs to BigInteger list + List benRegIds = new ArrayList<>(batchIds.size()); + for (Object[] idRow : batchIds) { + BigInteger benRegId = convertToBigInteger(idRow[0]); + if (benRegId != null) { + benRegIds.add(benRegId); + } + } - if (idObj instanceof BigInteger) { - benRegId = (BigInteger) idObj; - } else if (idObj instanceof Long) { - benRegId = BigInteger.valueOf((Long) idObj); - } else { - throw new IllegalArgumentException( - "Unsupported benRegId type: " + idObj.getClass()); + // STEP 3: BATCH FETCH complete data WITH ABHA (CRITICAL OPTIMIZATION) + // This single call replaces thousands of individual database queries + logger.info("Batch fetching complete data with ABHA for {} beneficiaries...", benRegIds.size()); + List documents = documentDataService.getBeneficiariesBatch(benRegIds); + logger.info("Retrieved {} complete documents", documents.size()); + + // STEP 4: Count ABHA enriched documents and add to ES batch + for (BeneficiaryDocument doc : documents) { + if (doc != null && doc.getBenId() != null) { + + // Track ABHA enrichment + if (doc.getHealthID() != null || doc.getAbhaID() != null) { + abhaEnrichedCount.incrementAndGet(); + logger.debug("Document {} has ABHA: healthID={}, abhaID={}", + doc.getBenId(), doc.getHealthID(), doc.getAbhaID()); } - // Fetch beneficiary details DIRECTLY from database - BeneficiariesDTO benDTO = beneficiaryDataService.getBeneficiaryFromDatabase(benRegId); - - if (benDTO != null) { - BeneficiaryDocument doc = convertToDocument(benDTO); - - if (doc != null && doc.getBenId() != null) { - esBatch.add(doc); - - // Send to ES when batch is full - if (esBatch.size() >= ES_BULK_SIZE) { - int indexed = bulkIndexDocuments(esBatch); - result.addSuccess(indexed); - result.addFailure(esBatch.size() - indexed); - - int current = processedCount.addAndGet(esBatch.size()); - logger.info("Progress: {}/{} ({} %) - Indexed: {}, Failed: {}", - current, totalCount, - String.format("%.2f", (current * 100.0) / totalCount), - indexed, esBatch.size() - indexed); - - esBatch.clear(); - } - } else { - logger.warn("Skipping beneficiary with null benId: benRegId={}", benRegId); - result.addFailure(); - } - } else { - logger.warn("No details found for benRegId: {}", benRegId); - result.addFailure(); - } + esBatch.add(doc); + + // Bulk index when batch is full + if (esBatch.size() >= ES_BULK_SIZE) { + int indexed = bulkIndexDocuments(esBatch); + result.addSuccess(indexed); + result.addFailure(esBatch.size() - indexed); + + int current = processedCount.addAndGet(esBatch.size()); + logProgress(current, totalCount, abhaEnrichedCount.get(), startTime); - } catch (Exception e) { - logger.error("Error processing beneficiary in batch: {}", e.getMessage(), e); + esBatch.clear(); + } + } else { result.addFailure(); } } - offset += BATCH_SIZE; - batchCounter++; + long chunkTime = System.currentTimeMillis() - chunkStart; + logger.info("Chunk processed in {}ms ({} docs/sec)", + chunkTime, + (batchIds.size() * 1000) / Math.max(chunkTime, 1)); - // Pause after every N batches to let connections stabilize - if (batchCounter % PAUSE_AFTER_BATCHES == 0) { - logger.info("Completed {} batches. Pausing for 2 seconds...", batchCounter); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - logger.warn("Sleep interrupted: {}", e.getMessage()); - } - } + offset += DB_FETCH_SIZE; + + // Brief pause to prevent overwhelming the system + Thread.sleep(50); } // Index remaining documents @@ -177,15 +145,21 @@ public SyncResult syncAllBeneficiaries() { processedCount.addAndGet(esBatch.size()); } - logger.info("Sync completed successfully!"); + long totalTime = System.currentTimeMillis() - startTime; + double docsPerSecond = (processedCount.get() * 1000.0) / totalTime; + + logger.info("SYNC COMPLETED SUCCESSFULLY!"); logger.info("Total Processed: {}", processedCount.get()); logger.info("Successfully Indexed: {}", result.getSuccessCount()); logger.info("Failed: {}", result.getFailureCount()); + logger.info("ABHA Enriched: {} ({} %)", + abhaEnrichedCount.get(), + String.format("%.2f", (abhaEnrichedCount.get() * 100.0) / processedCount.get())); + logger.info("Total Time: {} seconds ({} minutes)", totalTime / 1000, totalTime / 60000); + logger.info("Throughput: {:.2f} documents/second", docsPerSecond); } catch (Exception e) { - logger.error("========================================"); - logger.error("CRITICAL ERROR during full sync: {}", e.getMessage(), e); - logger.error("========================================"); + logger.error("CRITICAL ERROR during sync: {}", e.getMessage(), e); result.setError(e.getMessage()); } @@ -193,64 +167,61 @@ public SyncResult syncAllBeneficiaries() { } /** - * Sync a single beneficiary by BenRegId - * Uses direct database access to avoid Elasticsearch circular dependency + * Fetch batch with automatic retry on connection errors */ - public boolean syncSingleBeneficiary(String benRegId) { - try { - - BigInteger benRegIdBig = new BigInteger(benRegId); - - // Check if beneficiary exists in database first using transactional wrapper - boolean exists = transactionalWrapper.existsByBenRegId(benRegIdBig); - if (!exists) { - logger.error("Beneficiary does not exist in database}"); - return false; - } - - logger.info("Beneficiary exists in database. Fetching details..."); - - // Get beneficiary DIRECTLY from database (not through IdentityService) - BeneficiariesDTO benDTO = beneficiaryDataService.getBeneficiaryFromDatabase(benRegIdBig); - - if (benDTO == null) { - logger.error("Failed to fetch beneficiary details from database"); - return false; - } - - logger.info("Beneficiary details fetched successfully"); - logger.info("BenRegId: {}, Name: {} {}", - benDTO.getBenRegId(), - benDTO.getBeneficiaryDetails() != null ? benDTO.getBeneficiaryDetails().getFirstName() : "N/A", - benDTO.getBeneficiaryDetails() != null ? benDTO.getBeneficiaryDetails().getLastName() : "N/A"); - - // Convert to Elasticsearch document - BeneficiaryDocument doc = convertToDocument(benDTO); - - if (doc == null || doc.getBenId() == null) { - logger.error("Failed to convert beneficiary to document"); - return false; + private List fetchBatchWithRetry(int offset, int limit) { + int maxRetries = 3; + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + return transactionalWrapper.getBeneficiaryIdsBatch(offset, limit); + } catch (Exception e) { + logger.warn("Database fetch error (attempt {}/{}): {}", attempt, maxRetries, e.getMessage()); + if (attempt == maxRetries) { + throw new RuntimeException("Failed to fetch batch after " + maxRetries + " attempts", e); + } + try { + Thread.sleep(1000 * attempt); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry", ie); + } } + } + return null; + } - logger.info("Document created. Indexing to Elasticsearch..."); - logger.info("Document ID: {}, Index: {}", doc.getBenId(), beneficiaryIndex); - - // Index to Elasticsearch - esClient.index(i -> i - .index(beneficiaryIndex) - .id(doc.getBenId()) - .document(doc)); - - logger.info("SUCCESS! Beneficiary synced to Elasticsearch"); - - return true; - - } catch (Exception e) { - logger.error("ERROR syncing beneficiary {}: {}", benRegId, e.getMessage(), e); - return false; + /** + * Convert various ID types to BigInteger + */ + private BigInteger convertToBigInteger(Object idObj) { + if (idObj instanceof BigInteger) { + return (BigInteger) idObj; + } else if (idObj instanceof Long) { + return BigInteger.valueOf((Long) idObj); + } else if (idObj instanceof Integer) { + return BigInteger.valueOf((Integer) idObj); + } else { + logger.warn("Unsupported ID type: {}", idObj != null ? idObj.getClass() : "null"); + return null; } } + /** + * Progress logging with ETA and ABHA count + */ + private void logProgress(int current, long total, int abhaCount, long startTime) { + double progress = (current * 100.0) / total; + long elapsed = System.currentTimeMillis() - startTime; + long estimatedTotal = (long) (elapsed / (progress / 100.0)); + long remaining = estimatedTotal - elapsed; + + logger.info("Progress: {}/{} ({:.2f}%) | ABHA: {} | Elapsed: {}m | ETA: {}m | Speed: {:.0f} docs/sec", + current, total, progress, abhaCount, + elapsed / 60000, + remaining / 60000, + (current * 1000.0) / elapsed); + } + /** * Bulk index documents to Elasticsearch */ @@ -277,11 +248,9 @@ private int bulkIndexDocuments(List documents) { int successCount = 0; if (result.errors()) { - logger.warn("Bulk indexing had some errors"); for (BulkResponseItem item : result.items()) { if (item.error() != null) { - logger.error("Error indexing document {}: {}", - item.id(), item.error().reason()); + logger.error("Error indexing document {}: {}", item.id(), item.error().reason()); } else { successCount++; } @@ -299,75 +268,54 @@ private int bulkIndexDocuments(List documents) { } /** - * Convert BeneficiariesDTO to BeneficiaryDocument + * Sync a single beneficiary with ABHA */ - private BeneficiaryDocument convertToDocument(BeneficiariesDTO dto) { - if (dto == null) { - logger.warn("Cannot convert null DTO to document"); - return null; - } - + public boolean syncSingleBeneficiary(String benRegId) { try { - BeneficiaryDocument doc = new BeneficiaryDocument(); - - // BenId (use benRegId as primary identifier) - if (dto.getBenRegId() != null) { - BigInteger benRegId = (BigInteger) dto.getBenRegId(); - doc.setBenId(benRegId.toString()); - doc.setBenRegId(benRegId.longValue()); - } else if (dto.getBenId() != null) { - doc.setBenId(dto.getBenId().toString()); - if (dto.getBenId() instanceof BigInteger) { - doc.setBenRegId(((BigInteger) dto.getBenId()).longValue()); - } - } else { - logger.warn("Beneficiary has no valid ID!"); - return null; + BigInteger benRegIdBig = new BigInteger(benRegId); + + // Check existence + boolean exists = transactionalWrapper.existsByBenRegId(benRegIdBig); + if (!exists) { + logger.error("Beneficiary does not exist in database: {}", benRegId); + return false; } - // Phone number - doc.setPhoneNum(dto.getPreferredPhoneNum()); + logger.info("Beneficiary exists in database. Fetching details with ABHA..."); + + // Fetch document using batch service (includes ABHA) + BeneficiaryDocument doc = documentDataService.getBeneficiaryFromDatabase(benRegIdBig); - // Beneficiary Details (from nested DTO) - if (dto.getBeneficiaryDetails() != null) { - BenDetailDTO benDetails = dto.getBeneficiaryDetails(); - doc.setFirstName(benDetails.getFirstName()); - doc.setLastName(benDetails.getLastName()); - doc.setAge(benDetails.getBeneficiaryAge()); - doc.setGender(benDetails.getGender()); + if (doc == null || doc.getBenId() == null) { + logger.error("Failed to fetch beneficiary document"); + return false; } - // ===== EXTRACT ABHA DETAILS ===== - if (dto.getAbhaDetails() != null && !dto.getAbhaDetails().isEmpty()) { - AbhaAddressDTO abhaDTO = dto.getAbhaDetails().get(0); - if (abhaDTO.getHealthID() != null) { - doc.setHealthID(abhaDTO.getHealthID()); - logger.debug("Set healthID={} for benRegId={}", abhaDTO.getHealthID(), doc.getBenRegId()); - } + logger.info("Document created with ABHA. healthID={}, abhaID={}", + doc.getHealthID(), doc.getAbhaID()); - if (abhaDTO.getHealthIDNumber() != null) { - doc.setAbhaID(abhaDTO.getHealthIDNumber()); - logger.debug("Set abhaID={} for benRegId={}", abhaDTO.getHealthIDNumber(), doc.getBenRegId()); - } - } - logger.debug("Successfully converted DTO to document: benId={}", doc.getBenId()); - return doc; + // Index to Elasticsearch + esClient.index(i -> i + .index(beneficiaryIndex) + .id(doc.getBenId()) + .document(doc)); + + logger.info("SUCCESS! Beneficiary {} synced to Elasticsearch with ABHA", benRegId); + return true; } catch (Exception e) { - logger.error("Error converting DTO to document: {}", e.getMessage(), e); - return null; + logger.error("ERROR syncing beneficiary {}: {}", benRegId, e.getMessage(), e); + return false; } } /** - * Verify sync by checking document count + * Check sync status */ public SyncStatus checkSyncStatus() { try { long dbCount = transactionalWrapper.countActiveBeneficiaries(); - - long esCount = esClient.count(c -> c - .index(beneficiaryIndex)).count(); + long esCount = esClient.count(c -> c.index(beneficiaryIndex)).count(); SyncStatus status = new SyncStatus(); status.setDatabaseCount(dbCount); @@ -376,7 +324,6 @@ public SyncStatus checkSyncStatus() { status.setMissingCount(dbCount - esCount); logger.info("Sync Status - DB: {}, ES: {}, Missing: {}", dbCount, esCount, dbCount - esCount); - return status; } catch (Exception e) { @@ -425,17 +372,11 @@ public void setError(String error) { @Override public String toString() { - return "SyncResult{" + - "successCount=" + successCount + - ", failureCount=" + failureCount + - ", error='" + error + '\'' + - '}'; + return "SyncResult{successCount=" + successCount + ", failureCount=" + failureCount + + ", error='" + error + "'}"; } } - /** - * Status class to track sync verification - */ public static class SyncStatus { private long databaseCount; private long elasticsearchCount; @@ -485,13 +426,8 @@ public void setError(String error) { @Override public String toString() { - return "SyncStatus{" + - "databaseCount=" + databaseCount + - ", elasticsearchCount=" + elasticsearchCount + - ", synced=" + synced + - ", missingCount=" + missingCount + - ", error='" + error + '\'' + - '}'; + return "SyncStatus{databaseCount=" + databaseCount + ", elasticsearchCount=" + elasticsearchCount + + ", synced=" + synced + ", missingCount=" + missingCount + ", error='" + error + "'}"; } } } \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8774780..5001e41 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -21,28 +21,33 @@ spring.datasource.hikari.data-source-properties.tcpKeepAlive=true # spring.datasource.hikari.test-on-borrow=true # Reduce batch size for faster feedback -spring.jpa.properties.hibernate.jdbc.batch_size=50 +spring.jpa.properties.hibernate.jdbc.fetch_size=10000 +spring.jpa.properties.hibernate.jdbc.batch_size=5000 spring.jpa.properties.hibernate.order_inserts=true spring.jpa.properties.hibernate.order_updates=true # Query timeout -spring.jpa.properties.hibernate.query.timeout=30000 +spring.jpa.properties.hibernate.query.timeout=60000 -spring.datasource.hikari.maximum-pool-size=20 -spring.datasource.hikari.minimum-idle=10 +spring.datasource.hikari.maximum-pool-size=30 +spring.datasource.hikari.minimum-idle=15 spring.datasource.hikari.connection-timeout=30000 spring.datasource.hikari.idle-timeout=300000 spring.datasource.hikari.max-lifetime=600000 -spring.datasource.hikari.keepalive-tquickime=120000 -spring.datasource.hikari.validationTimeout=5000 +spring.datasource.hikari.keepalive-time=120000 +spring.datasource.hikari.validation-timeout=5000 +spring.datasource.hikari.leak-detection-threshold=60000 -spring.datasource.hikari.health-check-properties.mysql5Validation=true -spring.datasource.hikari.data-source-properties.cachePrepStmts=true -spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250 -spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048 +ring.datasource.hikari.data-source-properties.cachePrepStmts=true +spring.datasource.hikari.data-source-properties.prepStmtCacheSize=500 +spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=4096 spring.datasource.hikari.data-source-properties.useServerPrepStmts=true +spring.datasource.hikari.data-source-properties.autoReconnect=true spring.datasource.hikari.data-source-properties.tcpKeepAlive=true +spring.datasource.hikari.data-source-properties.useSSL=false + +spring.datasource.hikari.health-check-properties.mysql5Validation=true #Below lines are added for security reasons spring.session.store-type=redis @@ -75,45 +80,47 @@ spring.main.allow-circular-references=true jwt.access.expiration=86400000 jwt.refresh.expiration=604800000 -# Connection pool settings -elasticsearch.connection.timeout=5000 -elasticsearch.socket.timeout=60000 -elasticsearch.max.retry.timeout=60000 +# Connection timeouts +elasticsearch.connection.timeout=10000 +elasticsearch.socket.timeout=120000 +elasticsearch.max.retry.timeout=120000 -# Connection pooling -elasticsearch.max.connections=100 -elasticsearch.max.connections.per.route=50 +# Connection pooling - INCREASED for bulk operations +elasticsearch.max.connections=200 +elasticsearch.max.connections.per.route=100 -# Request Configuration -elasticsearch.request.timeout=30000 +# Request configuration +elasticsearch.request.timeout=60000 elasticsearch.max.result.window=10000 -# Bulk Indexing Performance -elasticsearch.bulk.size=100 -elasticsearch.bulk.concurrent.requests=4 -elasticsearch.bulk.flush.interval=10s +# Bulk indexing - OPTIMIZED for maximum throughput +elasticsearch.bulk.size=2000 +elasticsearch.bulk.concurrent.requests=8 +elasticsearch.bulk.flush.interval=30s -# Search Performance +# Search performance (unchanged - maintains fast search) elasticsearch.search.default.size=100 elasticsearch.search.max.size=500 elasticsearch.search.timeout=5s -# Query Cache Settings +# Query cache (for search performance) elasticsearch.query.cache.enabled=true elasticsearch.query.cache.size=10% -# Request Cache Settings +# Request cache elasticsearch.request.cache.enabled=true -# Circuit Breaker (Prevent OOM) +# Circuit breaker elasticsearch.circuit.breaker.enabled=true elasticsearch.circuit.breaker.limit=95% -# Thread Pool for Async Operations -elasticsearch.async.thread.pool.size=10 -elasticsearch.async.thread.pool.queue.size=1000 +# Async operations thread pool +elasticsearch.async.thread.pool.size=20 +elasticsearch.async.thread.pool.queue.size=5000 -# Logging +# LOGGING (Reduce noise during sync) logging.level.com.iemr.common.identity.service.elasticsearch=INFO logging.level.org.elasticsearch=WARN +logging.level.org.hibernate.SQL=WARN +logging.level.com.zaxxer.hikari=INFO From f7384c98b80413969de354d0860a5d40110208d8 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 20 Jan 2026 14:42:06 +0530 Subject: [PATCH 05/14] fix: get abha created date --- .../elasticsearch/BeneficiaryDocument.java | 3 + .../identity/dto/BeneficiariesESDTO.java | 3 + .../BeneficiaryDocumentDataService.java | 13 ++++- .../ElasticsearchIndexingService.java | 1 + .../elasticsearch/ElasticsearchService.java | 58 ++++++++++++++----- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java b/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java index d8b3b01..aad9011 100644 --- a/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java +++ b/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java @@ -58,6 +58,9 @@ public class BeneficiaryDocument { @JsonProperty("abhaID") private String abhaID; + @JsonProperty("abhaCreatedDate") + private String abhaCreatedDate; + @JsonProperty("familyID") private String familyID; diff --git a/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java b/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java index 969f8af..6238e64 100644 --- a/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java +++ b/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java @@ -77,6 +77,9 @@ public class BeneficiariesESDTO { @JsonProperty("abhaID") private String abhaID; + @JsonProperty("abhaCreatedDate") + private String abhaCreatedDate; + @JsonProperty("familyID") private String familyID; diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java index e64ddc6..763890a 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java @@ -65,6 +65,7 @@ public List getBeneficiariesBatch(List benRegId if (abhaData != null) { doc.setHealthID(abhaData.getHealthID()); doc.setAbhaID(abhaData.getHealthIDNumber()); + doc.setAbhaCreatedDate(abhaData.getAbhaCreatedDate()); logger.info("Enriched benRegId={} with healthID={}, abhaID={}", doc.getBenRegId(), doc.getHealthID(), doc.getAbhaID()); } else { @@ -148,7 +149,8 @@ private Map batchFetchAbhaDetails(List benRegIds) { abhaData.setHealthID(record[1] != null ? record[1].toString() : null); abhaData.setHealthIDNumber(record[2] != null ? record[2].toString() : null); abhaData.setAuthenticationMode(record[3] != null ? record[3].toString() : null); - + abhaData.setAbhaCreatedDate(record[4] !=null ? record[4].toString() : null); + abhaMap.put(benRegId, abhaData); } } catch (Exception e) { @@ -343,6 +345,7 @@ private static class AbhaData { private String healthID; private String healthIDNumber; private String authenticationMode; + private String abhaCreatedDate; public String getHealthID() { return healthID; @@ -367,6 +370,14 @@ public String getAuthenticationMode() { public void setAuthenticationMode(String authenticationMode) { this.authenticationMode = authenticationMode; } + + public String getAbhaCreatedDate() { + return abhaCreatedDate; + } + + public void setAbhaCreatedDate(String abhaCreatedDate) { + this.abhaCreatedDate = abhaCreatedDate; + } } } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java index 2b50c49..44d13d3 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java @@ -120,6 +120,7 @@ public void createIndexWithMapping() throws Exception { .properties("healthID", Property.of(p -> p.keyword(k -> k))) .properties("abhaID", Property.of(p -> p.keyword(k -> k))) + .properties("abhaCreatedDate", Property.of(p -> p.keyword(k -> k))) .properties("familyID", Property.of(p -> p.keyword(k -> k))) // Geographic fields diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index b0836cf..4e03809 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -18,6 +18,9 @@ import java.math.BigDecimal; import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; import co.elastic.clients.elasticsearch._types.SortOrder; @@ -166,13 +169,15 @@ public List> universalSearch(String query, Integer userId) { .filter(f -> f .includes("benRegId", "beneficiaryID", "firstName", "lastName", "genderID", "genderName", "dOB", "phoneNum", - "stateID", "districtID", "blockID", "villageID", "healthID", "abhaID", "familyID", - "fatherName", "spouseName", "age", "createdBy", "createdDate", - "lastModDate", "benAccountID", "districtName", "blockName", - "villageName", "pinCode", "servicePointID", "servicePointName", - "parkingPlaceID", "permStateID", "permStateName", "permDistrictID", - "permDistrictName", "permBlockID", "permBlockName", "permVillageID", - "permVillageName"))) + "stateID", "districtID", "blockID", "villageID", "healthID", "abhaID", + "abhaCreatedDate", + "familyID", + "fatherName", "spouseName", "age", "createdBy", "createdDate", + "lastModDate", "benAccountID", "districtName", "blockName", + "villageName", "pinCode", "servicePointID", "servicePointName", + "parkingPlaceID", "permStateID", "permStateName", "permDistrictID", + "permDistrictName", "permBlockID", "permBlockName", "permVillageID", + "permVillageName"))) , BeneficiariesESDTO.class); @@ -182,12 +187,12 @@ public List> universalSearch(String query, Integer userId) { query); if (!response.hits().hits().isEmpty()) { - BeneficiariesESDTO firstResult = response.hits().hits().get(0).source(); - logger.info("First result - benRegId: {}, healthID: {}, abhaID: {}", - firstResult.getBenRegId(), - firstResult.getHealthID(), - firstResult.getAbhaID()); -} + BeneficiariesESDTO firstResult = response.hits().hits().get(0).source(); + logger.info("First result - benRegId: {}, healthID: {}, abhaID: {}", + firstResult.getBenRegId(), + firstResult.getHealthID(), + firstResult.getAbhaID()); + } if (response.hits().hits().isEmpty()) { logger.info("No results in ES, using database fallback"); @@ -500,7 +505,7 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat return null; } - logger.info("ESDATA="+esData.getAbhaID()); + logger.info("ESDATA=" + esData.getAbhaID()); Map result = new HashMap<>(); @@ -522,10 +527,31 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat result.put("lastModDate", esData.getLastModDate()); result.put("benAccountID", esData.getBenAccountID()); - result.put("healthID", esData.getHealthID()); - result.put("abhaID", esData.getAbhaID()); result.put("familyID", esData.getFamilyID()); + List> abhaDetails = new ArrayList<>(); + if (esData.getHealthID() != null || esData.getAbhaID() != null) { + Map abhaDetail = new HashMap<>(); + abhaDetail.put("healthIDNumber", esData.getAbhaID()); + abhaDetail.put("healthID", esData.getAbhaID()); + if (esData.getAbhaCreatedDate() != null) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S"); + + LocalDateTime localDateTime = LocalDateTime.parse(esData.getAbhaCreatedDate(), formatter); + + long createdDateMillis = localDateTime + .atZone(ZoneId.of("Asia/Kolkata")) + .toInstant() + .toEpochMilli(); + + abhaDetail.put("createdDate", createdDateMillis); + } + + abhaDetail.put("beneficiaryRegID", esData.getBenRegId()); + abhaDetails.add(abhaDetail); + } + result.put("abhaDetails", abhaDetails); + Map mGender = new HashMap<>(); mGender.put("genderID", esData.getGenderID()); mGender.put("genderName", esData.getGenderName()); From 6cfa6e09710eb817fe40eacc01b58f5ebc1ab326 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 20 Jan 2026 16:15:07 +0530 Subject: [PATCH 06/14] fix: state issue --- .../java/com/iemr/common/identity/repo/BenMappingRepo.java | 3 ++- .../identity/service/elasticsearch/ElasticsearchService.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/repo/BenMappingRepo.java b/src/main/java/com/iemr/common/identity/repo/BenMappingRepo.java index 33db1d9..9c20f49 100644 --- a/src/main/java/com/iemr/common/identity/repo/BenMappingRepo.java +++ b/src/main/java/com/iemr/common/identity/repo/BenMappingRepo.java @@ -180,7 +180,8 @@ MBeneficiarymapping getWithVanSerialNoVanID(@Param("vanSerialNo") BigInteger van "UNIX_TIMESTAMP(m.LastModDate) * 1000, " + // 13 "m.BenAccountID, " + // 14 "contact.PreferredPhoneNum, " + // 15 - "h.HealthID, " + "h.HealthIDNumber, " + "fam.BenFamilyMapId, " + + // "h.HealthID, " + "h.HealthIDNumber, " + + "fam.BenFamilyMapId, " + "addr.CurrStateId, " + // 19 "addr.CurrState, " + // 20 "addr.CurrDistrictId, " + // 21 diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index 4e03809..8f2d58a 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -169,7 +169,7 @@ public List> universalSearch(String query, Integer userId) { .filter(f -> f .includes("benRegId", "beneficiaryID", "firstName", "lastName", "genderID", "genderName", "dOB", "phoneNum", - "stateID", "districtID", "blockID", "villageID", "healthID", "abhaID", + "stateID", "stateName", "districtID", "blockID", "villageID", "healthID", "abhaID", "abhaCreatedDate", "familyID", "fatherName", "spouseName", "age", "createdBy", "createdDate", From 8673ea6c78b08eed4b01ef11c470a5b2f8c952a9 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 20 Jan 2026 20:33:11 +0530 Subject: [PATCH 07/14] fix: village issue --- .../elasticsearch/ElasticsearchService.java | 544 ++++++++++-------- 1 file changed, 290 insertions(+), 254 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index 8f2d58a..14f9122 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -510,142 +510,167 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat Map result = new HashMap<>(); try { - // Basic fields from ES - result.put("beneficiaryRegID", esData.getBenRegId()); - result.put("beneficiaryID", esData.getBeneficiaryID()); - result.put("firstName", esData.getFirstName()); - result.put("lastName", esData.getLastName()); - result.put("genderID", esData.getGenderID()); - result.put("genderName", esData.getGenderName()); - result.put("dob", esData.getDOB()); - result.put("dOB", esData.getDOB()); - result.put("age", esData.getAge()); - result.put("fatherName", esData.getFatherName() != null ? esData.getFatherName() : ""); - result.put("spouseName", esData.getSpouseName() != null ? esData.getSpouseName() : ""); - result.put("createdBy", esData.getCreatedBy()); - result.put("createdDate", esData.getCreatedDate()); - result.put("lastModDate", esData.getLastModDate()); - result.put("benAccountID", esData.getBenAccountID()); - - result.put("familyID", esData.getFamilyID()); - - List> abhaDetails = new ArrayList<>(); - if (esData.getHealthID() != null || esData.getAbhaID() != null) { - Map abhaDetail = new HashMap<>(); - abhaDetail.put("healthIDNumber", esData.getAbhaID()); - abhaDetail.put("healthID", esData.getAbhaID()); - if (esData.getAbhaCreatedDate() != null) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S"); - - LocalDateTime localDateTime = LocalDateTime.parse(esData.getAbhaCreatedDate(), formatter); - - long createdDateMillis = localDateTime - .atZone(ZoneId.of("Asia/Kolkata")) - .toInstant() - .toEpochMilli(); - - abhaDetail.put("createdDate", createdDateMillis); - } - - abhaDetail.put("beneficiaryRegID", esData.getBenRegId()); - abhaDetails.add(abhaDetail); - } - result.put("abhaDetails", abhaDetails); - - Map mGender = new HashMap<>(); - mGender.put("genderID", esData.getGenderID()); - mGender.put("genderName", esData.getGenderName()); - result.put("m_gender", mGender); - - Map demographics = new HashMap<>(); - demographics.put("beneficiaryRegID", esData.getBenRegId()); - demographics.put("stateID", esData.getStateID()); - demographics.put("stateName", esData.getStateName()); - demographics.put("districtID", esData.getDistrictID()); - demographics.put("districtName", esData.getDistrictName()); - demographics.put("blockID", esData.getBlockID()); - demographics.put("blockName", esData.getBlockName()); - demographics.put("villageID", esData.getVillageID()); - demographics.put("villageName", esData.getVillageName()); - demographics.put("districtBranchID", null); - demographics.put("districtBranchName", null); - demographics.put("parkingPlaceID", esData.getParkingPlaceID()); - demographics.put("servicePointID", esData.getServicePointID()); - demographics.put("servicePointName", esData.getServicePointName()); - demographics.put("createdBy", esData.getCreatedBy()); - - Map mState = new HashMap<>(); - mState.put("stateID", esData.getStateID()); - mState.put("stateName", esData.getStateName()); - mState.put("stateCode", null); - mState.put("countryID", 1); - demographics.put("m_state", mState); - - Map mDistrict = new HashMap<>(); - mDistrict.put("districtID", esData.getDistrictID()); - mDistrict.put("districtName", esData.getDistrictName()); - mDistrict.put("stateID", esData.getStateID()); - demographics.put("m_district", mDistrict); - - Map mBlock = new HashMap<>(); - mBlock.put("blockID", esData.getBlockID()); - mBlock.put("blockName", esData.getBlockName()); - mBlock.put("districtID", esData.getDistrictID()); - mBlock.put("stateID", esData.getStateID()); - demographics.put("m_districtblock", mBlock); - - Map mBranch = new HashMap<>(); - mBranch.put("districtBranchID", null); - mBranch.put("blockID", esData.getBlockID()); - mBranch.put("villageName", esData.getVillageName()); - mBranch.put("pinCode", esData.getPinCode()); - demographics.put("m_districtbranchmapping", mBranch); - - result.put("i_bendemographics", demographics); - - List> benPhoneMaps = new ArrayList<>(); - if (esData.getPhoneNum() != null && !esData.getPhoneNum().isEmpty()) { - Map phoneMap = new HashMap<>(); - phoneMap.put("benPhMapID", 1L); - phoneMap.put("benificiaryRegID", esData.getBenRegId()); - phoneMap.put("parentBenRegID", esData.getBenRegId()); - phoneMap.put("benRelationshipID", 1); - phoneMap.put("phoneNo", esData.getPhoneNum()); - - Map relationType = new HashMap<>(); - relationType.put("benRelationshipID", 1); - relationType.put("benRelationshipType", "Self"); - phoneMap.put("benRelationshipType", relationType); - - benPhoneMaps.add(phoneMap); + // Basic fields from ES + result.put("beneficiaryRegID", esData.getBenRegId()); + result.put("beneficiaryID", esData.getBeneficiaryID()); + result.put("firstName", esData.getFirstName()); + result.put("lastName", esData.getLastName() != null ? esData.getLastName() : ""); + result.put("genderID", esData.getGenderID()); + result.put("genderName", esData.getGenderName()); + result.put("dob", esData.getDOB()); + result.put("dOB", esData.getDOB()); + result.put("age", esData.getAge()); + result.put("actualAge", esData.getAge()); // NEW + result.put("ageUnits", "Years"); // NEW + + result.put("fatherName", esData.getFatherName() != null ? esData.getFatherName() : ""); + result.put("spouseName", esData.getSpouseName() != null ? esData.getSpouseName() : ""); + result.put("isHIVPos", esData.getIsHIVPos() != null ? esData.getIsHIVPos() : ""); + + result.put("createdBy", esData.getCreatedBy()); + result.put("createdDate", esData.getCreatedDate()); + result.put("lastModDate", esData.getLastModDate()); + result.put("benAccountID", esData.getBenAccountID()); + + // Family ID - use both formats + result.put("familyID", esData.getFamilyID()); + result.put("familyId", esData.getFamilyID()); // NEW - alternate key + + // Marital status - NEW + result.put("maritalStatusName", null); // TODO: Add to ES if available + result.put("maritalStatus", new HashMap<>()); + + // Head of family - NEW (add these fields to BeneficiariesESDTO if available) + result.put("headOfFamily_RelationID", null); + result.put("headOfFamily_Relation", null); + + // ABHA Details + List> abhaDetails = new ArrayList<>(); + if (esData.getHealthID() != null || esData.getAbhaID() != null) { + Map abhaDetail = new HashMap<>(); + abhaDetail.put("healthIDNumber", esData.getAbhaID()); + abhaDetail.put("healthID", esData.getAbhaID()); + if (esData.getAbhaCreatedDate() != null) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S"); + LocalDateTime localDateTime = LocalDateTime.parse(esData.getAbhaCreatedDate(), formatter); + long createdDateMillis = localDateTime + .atZone(ZoneId.of("Asia/Kolkata")) + .toInstant() + .toEpochMilli(); + abhaDetail.put("createdDate", createdDateMillis); } - result.put("benPhoneMaps", benPhoneMaps); - - result.put("isConsent", false); - result.put("m_title", new HashMap<>()); - result.put("maritalStatus", new HashMap<>()); - result.put("changeInSelfDetails", false); - result.put("changeInAddress", false); - result.put("changeInContacts", false); - result.put("changeInIdentities", false); - result.put("changeInOtherDetails", false); - result.put("changeInFamilyDetails", false); - result.put("changeInAssociations", false); - result.put("changeInBankDetails", false); - result.put("changeInBenImage", false); - result.put("is1097", false); - result.put("emergencyRegistration", false); - result.put("passToNurse", false); - result.put("beneficiaryIdentities", new ArrayList<>()); - - } catch (Exception e) { - logger.error("Error mapping ES result: {}", e.getMessage(), e); - return null; + abhaDetail.put("beneficiaryRegID", esData.getBenRegId()); + abhaDetails.add(abhaDetail); } - - return result; + result.put("abhaDetails", abhaDetails); + + // Gender object + Map mGender = new HashMap<>(); + mGender.put("genderID", esData.getGenderID()); + mGender.put("genderName", esData.getGenderName()); + result.put("m_gender", mGender); + + // Demographics - UPDATED with all fields + Map demographics = new HashMap<>(); + demographics.put("beneficiaryRegID", esData.getBenRegId()); + + // Current address + demographics.put("stateID", esData.getStateID()); + demographics.put("stateName", esData.getStateName()); + demographics.put("districtID", esData.getDistrictID()); + demographics.put("districtName", esData.getDistrictName()); + demographics.put("blockID", esData.getBlockID()); + demographics.put("blockName", esData.getBlockName()); + demographics.put("villageID", esData.getVillageID()); // FIXED + demographics.put("villageName", esData.getVillageName()); // FIXED + + demographics.put("districtBranchID", esData.getVillageID()); + demographics.put("districtBranchName", esData.getVillageName()); + demographics.put("parkingPlaceID", esData.getParkingPlaceID()); + demographics.put("servicePointID", esData.getServicePointID()); + demographics.put("servicePointName", esData.getServicePointName()); + demographics.put("createdBy", esData.getCreatedBy()); + + // State object + Map mState = new HashMap<>(); + mState.put("stateID", esData.getStateID()); + mState.put("stateName", esData.getStateName()); + mState.put("stateCode", null); + mState.put("countryID", 1); + demographics.put("m_state", mState); + + // District object + Map mDistrict = new HashMap<>(); + mDistrict.put("districtID", esData.getDistrictID()); + mDistrict.put("districtName", esData.getDistrictName()); + mDistrict.put("stateID", esData.getStateID()); + demographics.put("m_district", mDistrict); + + // Block object + Map mBlock = new HashMap<>(); + mBlock.put("blockID", esData.getBlockID()); + mBlock.put("blockName", esData.getBlockName()); + mBlock.put("districtID", esData.getDistrictID()); + mBlock.put("stateID", esData.getStateID()); + demographics.put("m_districtblock", mBlock); + + // Branch mapping object + Map mBranch = new HashMap<>(); + mBranch.put("districtBranchID", null); + mBranch.put("blockID", esData.getBlockID()); + mBranch.put("villageName", esData.getVillageName()); // FIXED + mBranch.put("pinCode", esData.getPinCode()); + demographics.put("m_districtbranchmapping", mBranch); + + result.put("i_bendemographics", demographics); + + // Phone maps + List> benPhoneMaps = new ArrayList<>(); + if (esData.getPhoneNum() != null && !esData.getPhoneNum().isEmpty()) { + Map phoneMap = new HashMap<>(); + phoneMap.put("benPhMapID", 1L); + phoneMap.put("benificiaryRegID", esData.getBenRegId()); + phoneMap.put("parentBenRegID", esData.getBenRegId()); + phoneMap.put("benRelationshipID", 1); + phoneMap.put("phoneNo", esData.getPhoneNum()); + + Map relationType = new HashMap<>(); + relationType.put("benRelationshipID", 1); + relationType.put("benRelationshipType", "Self"); + phoneMap.put("benRelationshipType", relationType); + + benPhoneMaps.add(phoneMap); + } + result.put("benPhoneMaps", benPhoneMaps); + + // Boolean flags + result.put("isConsent", false); + result.put("changeInSelfDetails", false); + result.put("changeInAddress", false); + result.put("changeInContacts", false); + result.put("changeInIdentities", false); + result.put("changeInOtherDetails", false); + result.put("changeInFamilyDetails", false); + result.put("changeInAssociations", false); + result.put("changeInBankDetails", false); + result.put("changeInBenImage", false); + result.put("is1097", false); + result.put("emergencyRegistration", false); + result.put("passToNurse", false); + + // Empty objects/arrays + result.put("m_title", new HashMap<>()); + result.put("maritalStatus", new HashMap<>()); + result.put("beneficiaryIdentities", new ArrayList<>()); + + } catch (Exception e) { + logger.error("Error mapping ES result: {}", e.getMessage(), e); + return null; } + return result; +} + /** * Direct database search as fallback */ @@ -666,130 +691,141 @@ private List> searchInDatabaseDirectly(String query) { /** * Map database result to expected API format */ - private Map mapToExpectedFormat(Object[] row) { - Map result = new HashMap<>(); - - try { - Long beneficiaryRegID = getLong(row[0]); - String beneficiaryID = getString(row[1]); - String firstName = getString(row[2]); - String lastName = getString(row[3]); - Integer genderID = getInteger(row[4]); - String genderName = getString(row[5]); - Date dob = getDate(row[6]); - Integer age = getInteger(row[7]); - String fatherName = getString(row[8]); - String spouseName = getString(row[9]); - String isHIVPos = getString(row[10]); - String createdBy = getString(row[11]); - Date createdDate = getDate(row[12]); - Long lastModDate = getLong(row[13]); - Long benAccountID = getLong(row[14]); - - Integer stateID = getInteger(row[15]); - String stateName = getString(row[16]); - Integer districtID = getInteger(row[17]); - String districtName = getString(row[18]); - Integer blockID = getInteger(row[19]); - String blockName = getString(row[20]); - String pinCode = getString(row[21]); - Integer servicePointID = getInteger(row[22]); - String servicePointName = getString(row[23]); - Integer parkingPlaceID = getInteger(row[24]); - String phoneNum = getString(row[25]); - - result.put("beneficiaryRegID", beneficiaryRegID); - result.put("beneficiaryID", beneficiaryID); - result.put("firstName", firstName); - result.put("lastName", lastName); - result.put("genderID", genderID); - result.put("genderName", genderName); - result.put("dOB", dob); - result.put("dob", dob); - result.put("age", age); - result.put("fatherName", fatherName != null ? fatherName : ""); - result.put("spouseName", spouseName != null ? spouseName : ""); - result.put("createdBy", createdBy); - result.put("createdDate", createdDate); - result.put("lastModDate", lastModDate); - result.put("benAccountID", benAccountID); - - Map mGender = new HashMap<>(); - mGender.put("genderID", genderID); - mGender.put("genderName", genderName); - result.put("m_gender", mGender); - - Map demographics = new HashMap<>(); - demographics.put("beneficiaryRegID", beneficiaryRegID); - demographics.put("stateID", stateID); - demographics.put("stateName", stateName); - demographics.put("districtID", districtID); - demographics.put("districtName", districtName); - demographics.put("blockID", blockID); - demographics.put("blockName", blockName); - demographics.put("districtBranchID", null); - demographics.put("districtBranchName", null); - demographics.put("parkingPlaceID", parkingPlaceID); - demographics.put("servicePointID", servicePointID); - demographics.put("servicePointName", servicePointName); - demographics.put("createdBy", createdBy); - - Map mState = new HashMap<>(); - mState.put("stateID", stateID); - mState.put("stateName", stateName); - mState.put("stateCode", null); - mState.put("countryID", 1); - demographics.put("m_state", mState); - - Map mDistrict = new HashMap<>(); - mDistrict.put("districtID", districtID); - mDistrict.put("districtName", districtName); - mDistrict.put("stateID", stateID); - demographics.put("m_district", mDistrict); - - Map mBlock = new HashMap<>(); - mBlock.put("blockID", blockID); - mBlock.put("blockName", blockName); - mBlock.put("districtID", districtID); - mBlock.put("stateID", stateID); - demographics.put("m_districtblock", mBlock); - - Map mBranch = new HashMap<>(); - mBranch.put("districtBranchID", null); - mBranch.put("blockID", blockID); - mBranch.put("villageName", null); - mBranch.put("pinCode", pinCode); - demographics.put("m_districtbranchmapping", mBranch); - - result.put("i_bendemographics", demographics); - - List> benPhoneMaps = fetchPhoneNumbers(beneficiaryRegID); - result.put("benPhoneMaps", benPhoneMaps); - - result.put("isConsent", false); - result.put("m_title", new HashMap<>()); - result.put("maritalStatus", new HashMap<>()); - result.put("changeInSelfDetails", false); - result.put("changeInAddress", false); - result.put("changeInContacts", false); - result.put("changeInIdentities", false); - result.put("changeInOtherDetails", false); - result.put("changeInFamilyDetails", false); - result.put("changeInAssociations", false); - result.put("changeInBankDetails", false); - result.put("changeInBenImage", false); - result.put("is1097", false); - result.put("emergencyRegistration", false); - result.put("passToNurse", false); - result.put("beneficiaryIdentities", new ArrayList<>()); - - } catch (Exception e) { - logger.error("Error mapping result: {}", e.getMessage(), e); - } - - return result; + private Map mapToExpectedFormat(Object[] row) { + Map result = new HashMap<>(); + + try { + Long beneficiaryRegID = getLong(row[0]); + String beneficiaryID = getString(row[1]); + String firstName = getString(row[2]); + String lastName = getString(row[3]); + Integer genderID = getInteger(row[4]); + String genderName = getString(row[5]); + Date dob = getDate(row[6]); + Integer age = getInteger(row[7]); + String fatherName = getString(row[8]); + String spouseName = getString(row[9]); + String isHIVPos = getString(row[10]); + String createdBy = getString(row[11]); + Date createdDate = getDate(row[12]); + Long lastModDate = getLong(row[13]); + Long benAccountID = getLong(row[14]); + + Integer stateID = getInteger(row[15]); + String stateName = getString(row[16]); + Integer districtID = getInteger(row[17]); + String districtName = getString(row[18]); + Integer blockID = getInteger(row[19]); + String blockName = getString(row[20]); + String pinCode = getString(row[21]); + Integer servicePointID = getInteger(row[22]); + String servicePointName = getString(row[23]); + Integer parkingPlaceID = getInteger(row[24]); + String phoneNum = getString(row[25]); + + Integer villageID = getInteger(row[26]); + String villageName = getString(row[27]); + + result.put("beneficiaryRegID", beneficiaryRegID); + result.put("beneficiaryID", beneficiaryID); + result.put("firstName", firstName); + result.put("lastName", lastName != null ? lastName : ""); + result.put("genderID", genderID); + result.put("genderName", genderName); + result.put("dOB", dob); + result.put("dob", dob); + result.put("age", age); + result.put("actualAge", age); // NEW + result.put("ageUnits", "Years"); // NEW + result.put("fatherName", fatherName != null ? fatherName : ""); + result.put("spouseName", spouseName != null ? spouseName : ""); + result.put("isHIVPos", isHIVPos != null ? isHIVPos : ""); + result.put("createdBy", createdBy); + result.put("createdDate", createdDate); + result.put("lastModDate", lastModDate); + result.put("benAccountID", benAccountID); + + // NEW fields + result.put("maritalStatusName", null); + result.put("maritalStatus", new HashMap<>()); + + Map mGender = new HashMap<>(); + mGender.put("genderID", genderID); + mGender.put("genderName", genderName); + result.put("m_gender", mGender); + + Map demographics = new HashMap<>(); + demographics.put("beneficiaryRegID", beneficiaryRegID); + demographics.put("stateID", stateID); + demographics.put("stateName", stateName); + demographics.put("districtID", districtID); + demographics.put("districtName", districtName); + demographics.put("blockID", blockID); + demographics.put("blockName", blockName); + demographics.put("villageID", villageID); + demographics.put("villageName", villageName); + demographics.put("districtBranchID", villageID); + demographics.put("districtBranchName", villageName); + demographics.put("parkingPlaceID", parkingPlaceID); + demographics.put("servicePointID", servicePointID); + demographics.put("servicePointName", servicePointName); + demographics.put("createdBy", createdBy); + + Map mState = new HashMap<>(); + mState.put("stateID", stateID); + mState.put("stateName", stateName); + mState.put("stateCode", null); + mState.put("countryID", 1); + demographics.put("m_state", mState); + + Map mDistrict = new HashMap<>(); + mDistrict.put("districtID", districtID); + mDistrict.put("districtName", districtName); + mDistrict.put("stateID", stateID); + demographics.put("m_district", mDistrict); + + Map mBlock = new HashMap<>(); + mBlock.put("blockID", blockID); + mBlock.put("blockName", blockName); + mBlock.put("districtID", districtID); + mBlock.put("stateID", stateID); + demographics.put("m_districtblock", mBlock); + + Map mBranch = new HashMap<>(); + mBranch.put("districtBranchID", null); + mBranch.put("blockID", blockID); + mBranch.put("villageName", villageName); // FIXED + mBranch.put("pinCode", pinCode); + demographics.put("m_districtbranchmapping", mBranch); + + result.put("i_bendemographics", demographics); + + List> benPhoneMaps = fetchPhoneNumbers(beneficiaryRegID); + result.put("benPhoneMaps", benPhoneMaps); + + result.put("isConsent", false); + result.put("m_title", new HashMap<>()); + result.put("maritalStatus", new HashMap<>()); + result.put("changeInSelfDetails", false); + result.put("changeInAddress", false); + result.put("changeInContacts", false); + result.put("changeInIdentities", false); + result.put("changeInOtherDetails", false); + result.put("changeInFamilyDetails", false); + result.put("changeInAssociations", false); + result.put("changeInBankDetails", false); + result.put("changeInBenImage", false); + result.put("is1097", false); + result.put("emergencyRegistration", false); + result.put("passToNurse", false); + result.put("beneficiaryIdentities", new ArrayList<>()); + + } catch (Exception e) { + logger.error("Error mapping result: {}", e.getMessage(), e); } + return result; +} /** * Fetch phone numbers for a beneficiary */ From a7a46ef2bfde9f4f7fa94247b0f9ff68fbbd3783 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 21 Jan 2026 14:19:19 +0530 Subject: [PATCH 08/14] fix: add abha details --- .../identity/service/IdentityService.java | 42 ++++- .../ElasticsearchMappingUpdateService.java | 178 ++++++++++++++++++ 2 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java diff --git a/src/main/java/com/iemr/common/identity/service/IdentityService.java b/src/main/java/com/iemr/common/identity/service/IdentityService.java index d107ae1..f113f27 100644 --- a/src/main/java/com/iemr/common/identity/service/IdentityService.java +++ b/src/main/java/com/iemr/common/identity/service/IdentityService.java @@ -95,7 +95,7 @@ import com.iemr.common.identity.service.elasticsearch.ElasticsearchService; import com.iemr.common.identity.utils.mapper.OutputMapper; import com.iemr.common.identity.utils.response.OutputResponse; - +import org.springframework.scheduling.annotation.Async; import org.springframework.beans.factory.annotation.Value; import com.iemr.common.identity.service.elasticsearch.BeneficiaryElasticsearchIndexUpdater; @@ -998,6 +998,9 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields MBeneficiarymapping benMapping = mappingRepo.findByBenRegIdOrderByBenMapIdAsc(identity.getBeneficiaryRegId()); + // Track if ANY change happened (determines if ES sync is needed) + boolean isDataUpdate = false; + // change in self details is implement here and other details here logger.debug("identity.getChangeInSelfDetails = " + identity.getChangeInSelfDetails()); logger.debug("identity.getChangeInOtherDetails = " + identity.getChangeInOtherDetails()); @@ -1066,6 +1069,7 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields mbDetl.setEmergencyRegistration(false); } detailRepo.save(mbDetl); + isDataUpdate = true; } } // edition in current emergency and permanent is implement below @@ -1092,6 +1096,7 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields */ logger.debug("Beneficiary address to update = " + OutputMapper.gson().toJson(mbAddr)); addressRepo.save(mbAddr); + isDataUpdate = true; } // edition in beneficiary contacts is updated here @@ -1118,6 +1123,8 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields */ logger.debug("Beneficiary contact to update = " + OutputMapper.gson().toJson(benCon)); contactRepo.save(benCon); + isDataUpdate = true; + } // change in identities are added here @@ -1160,6 +1167,8 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields } index++; + isDataUpdate = true; + } } @@ -1204,6 +1213,8 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields } index++; + isDataUpdate = true; + } } @@ -1230,6 +1241,8 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields */ logger.debug("Account to upsert = " + OutputMapper.gson().toJson(beneficiaryAccount)); accountRepo.save(beneficiaryAccount); + isDataUpdate = true; + } if (Boolean.TRUE.equals(identity.getChangeInBenImage())) { @@ -1254,6 +1267,8 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields logger.debug("Image to upsert = " + OutputMapper.gson().toJson(beneficiaryImage)); beneficiaryImage.setProcessed("N"); imageRepo.save(beneficiaryImage); + isDataUpdate = true; + } // Trigger async sync to Elasticsearch @@ -1261,8 +1276,29 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields logger.info("Triggering Elasticsearch sync for benRegId: {}", identity.getBeneficiaryRegId()); syncService.syncBeneficiaryAsync(identity.getBeneficiaryRegId()); } - logger.info("IdentityService.editIdentity - end. id = " + benMapping.getBenMapId()); + + // ***ABHA details changes *** + logger.debug("identity.getChangeInAbhaDetails = " + identity.getChangeInOtherDetails()); + if (Boolean.TRUE.equals(identity.getChangeInOtherDetails())) { + logger.info("ABHA details changed for benRegId: {}", identity.getBeneficiaryRegId()); + + isDataUpdate = true; + logger.info("ABHA details saved successfully"); + } + + if (isDataUpdate && identity.getBeneficiaryRegId() != null) { + logger.info("Changes detected - Triggering Elasticsearch sync for benRegId: {}", + identity.getBeneficiaryRegId()); + syncService.syncBeneficiaryAsync(identity.getBeneficiaryRegId()); + } else if (identity.getBeneficiaryRegId() != null) { + logger.info("No changes detected - Skipping ES sync for benRegId: {}", + identity.getBeneficiaryRegId()); } + + logger.info("IdentityService.editIdentity - end. id = " + benMapping.getBenMapId()); +} + + private MBeneficiarydetail convertIdentityEditDTOToMBeneficiarydetail(IdentityEditDTO dto) { MBeneficiarydetail beneficiarydetail = new MBeneficiarydetail(); @@ -2142,5 +2178,7 @@ public Long checkBenIDAvailabilityLocal() { return regIdRepo.countByProvisioned(false); } + + } diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java new file mode 100644 index 0000000..0b205fe --- /dev/null +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java @@ -0,0 +1,178 @@ +package com.iemr.common.identity.service.elasticsearch; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +/** + * Service to update Elasticsearch mappings without deleting the index + * Use this to add new fields to existing index + */ +@Service +public class ElasticsearchMappingUpdateService { + + private static final Logger logger = LoggerFactory.getLogger(ElasticsearchMappingUpdateService.class); + + @Autowired + private ElasticsearchClient esClient; + + @Value("${elasticsearch.index.beneficiary}") + private String beneficiaryIndex; + + /** + * Add ABHA-related fields to existing index mapping + * This does NOT require deleting the index or re-syncing data + * + * NOTE: You can only ADD new fields, you cannot CHANGE existing field types + */ + public void addAbhaFieldsToExistingIndex() throws Exception { + logger.info("Adding ABHA fields to existing index: {}", beneficiaryIndex); + + try { + // Check if index exists + boolean indexExists = esClient.indices().exists(e -> e.index(beneficiaryIndex)).value(); + + if (!indexExists) { + logger.error("Index {} does not exist. Create it first.", beneficiaryIndex); + throw new RuntimeException("Index does not exist: " + beneficiaryIndex); + } + + // Update mapping by adding new properties + esClient.indices().putMapping(pm -> pm + .index(beneficiaryIndex) + .properties("healthID", Property.of(p -> p.keyword(k -> k))) + .properties("healthIDNumber", Property.of(p -> p.keyword(k -> k))) + .properties("abhaID", Property.of(p -> p.keyword(k -> k))) + .properties("abhaCreatedDate", Property.of(p -> p.keyword(k -> k))) + .properties("abhaAuthMode", Property.of(p -> p.keyword(k -> k))) + + // Nested object for multiple ABHA addresses + .properties("abhaAddresses", Property.of(p -> p.nested(n -> n + .properties("healthID", Property.of(fp -> fp.keyword(k -> k))) + .properties("healthIDNumber", Property.of(fp -> fp.keyword(k -> k))) + .properties("authenticationMode", Property.of(fp -> fp.keyword(k -> k))) + .properties("createdDate", Property.of(fp -> fp.date(d -> d))) + ))) + + // Add villageID and villageName if missing + .properties("villageID", Property.of(p -> p.integer(i -> i))) + .properties("villageName", Property.of(p -> p.keyword(k -> k))) + ); + + logger.info("ABHA fields added successfully to index: {}", beneficiaryIndex); + logger.info("No data loss - existing documents are intact"); + logger.info("New fields will be populated when documents are updated/synced"); + + // Verify the update + verifyMappingUpdate(); + + } catch (Exception e) { + logger.error("Failed to add ABHA fields to index", e); + throw e; + } + } + + /** + * Verify that new fields were added successfully + */ + private void verifyMappingUpdate() throws Exception { + var mapping = esClient.indices().getMapping(g -> g.index(beneficiaryIndex)); + + logger.info("Current index mapping properties:"); + mapping.get(beneficiaryIndex).mappings().properties().keySet().forEach(field -> { + logger.info(" - {}", field); + }); + } + + /** + * Update existing documents with ABHA data (incremental sync) + * This updates only documents that have ABHA details in the database + * + * Call this after adding new fields to populate them with existing data + */ + public Map syncAbhaDataToExistingDocuments() { + logger.info("Starting incremental ABHA data sync to existing ES documents"); + + Map result = new HashMap<>(); + int successCount = 0; + int failureCount = 0; + + try { + // This will be handled by your existing BeneficiaryElasticsearchIndexUpdater + // which already fetches ABHA details from v_BenAdvanceSearchRepo + + // You can either: + // 1. Sync all beneficiaries (slow but complete) + // 2. Sync only beneficiaries with ABHA data (fast and targeted) + + logger.info("Use the existing syncAllBeneficiaries() method"); + logger.info("It will automatically populate new ABHA fields"); + + result.put("message", 0); + result.put("recommendation", 1); + + } catch (Exception e) { + logger.error("Error during ABHA data sync", e); + } + + result.put("success", successCount); + result.put("failed", failureCount); + return result; + } + + /** + * Get current index statistics and mapping info + */ + public Map getIndexInfo() throws Exception { + Map info = new HashMap<>(); + + // Get mapping + var mapping = esClient.indices().getMapping(g -> g.index(beneficiaryIndex)); + info.put("totalFields", mapping.get(beneficiaryIndex).mappings().properties().size()); + + // Get stats + var stats = esClient.indices().stats(s -> s.index(beneficiaryIndex)); + info.put("documentCount", stats.indices().get(beneficiaryIndex).primaries().docs().count()); + info.put("sizeInBytes", stats.indices().get(beneficiaryIndex).primaries().store().sizeInBytes()); + + // Get settings + var settings = esClient.indices().getSettings(g -> g.index(beneficiaryIndex)); + info.put("numberOfShards", settings.get(beneficiaryIndex).settings().index().numberOfShards()); + info.put("numberOfReplicas", settings.get(beneficiaryIndex).settings().index().numberOfReplicas()); + info.put("refreshInterval", settings.get(beneficiaryIndex).settings().index().refreshInterval().time()); + + // Check if ABHA fields exist + boolean hasAbhaFields = mapping.get(beneficiaryIndex).mappings().properties() + .containsKey("healthID") && mapping.get(beneficiaryIndex).mappings().properties() + .containsKey("abhaID"); + info.put("hasAbhaFields", hasAbhaFields); + + return info; + } + + /** + * Add any additional fields to index without deleting + * Use this template for future field additions + */ + public void addCustomFieldsToIndex(Map newFields) throws Exception { + logger.info("Adding {} custom fields to index: {}", newFields.size(), beneficiaryIndex); + + esClient.indices().putMapping(pm -> { + var builder = pm.index(beneficiaryIndex); + newFields.forEach((fieldName, property) -> { + builder.properties(fieldName, property); + logger.info(" Adding field: {}", fieldName); + }); + return builder; + }); + + logger.info("Custom fields added successfully"); + } +} \ No newline at end of file From 4aff852f546c86a36dd6d6f5a5774d9376457244 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 21 Jan 2026 22:37:23 +0530 Subject: [PATCH 09/14] fix: abha fix and refresh index --- .../identity/service/IdentityService.java | 20 +- .../elasticsearch/BeneficiaryDataService.java | 191 ------ .../BeneficiaryDocumentDataService.java | 60 +- .../BeneficiaryElasticsearchIndexUpdater.java | 47 +- .../ElasticsearchMappingUpdateService.java | 178 ------ .../elasticsearch/ElasticsearchService.java | 600 +++++++++--------- .../ElasticsearchSyncService.java | 3 +- 7 files changed, 376 insertions(+), 723 deletions(-) delete mode 100644 src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java delete mode 100644 src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java diff --git a/src/main/java/com/iemr/common/identity/service/IdentityService.java b/src/main/java/com/iemr/common/identity/service/IdentityService.java index f113f27..6db2101 100644 --- a/src/main/java/com/iemr/common/identity/service/IdentityService.java +++ b/src/main/java/com/iemr/common/identity/service/IdentityService.java @@ -1272,28 +1272,10 @@ public void editIdentity(IdentityEditDTO identity) throws MissingMandatoryFields } // Trigger async sync to Elasticsearch - if (identity.getBeneficiaryRegId() != null) { + if (isDataUpdate && identity.getBeneficiaryRegId() != null) { logger.info("Triggering Elasticsearch sync for benRegId: {}", identity.getBeneficiaryRegId()); syncService.syncBeneficiaryAsync(identity.getBeneficiaryRegId()); } - - // ***ABHA details changes *** - logger.debug("identity.getChangeInAbhaDetails = " + identity.getChangeInOtherDetails()); - if (Boolean.TRUE.equals(identity.getChangeInOtherDetails())) { - logger.info("ABHA details changed for benRegId: {}", identity.getBeneficiaryRegId()); - - isDataUpdate = true; - logger.info("ABHA details saved successfully"); - } - - if (isDataUpdate && identity.getBeneficiaryRegId() != null) { - logger.info("Changes detected - Triggering Elasticsearch sync for benRegId: {}", - identity.getBeneficiaryRegId()); - syncService.syncBeneficiaryAsync(identity.getBeneficiaryRegId()); - } else if (identity.getBeneficiaryRegId() != null) { - logger.info("No changes detected - Skipping ES sync for benRegId: {}", - identity.getBeneficiaryRegId()); - } logger.info("IdentityService.editIdentity - end. id = " + benMapping.getBenMapId()); } diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java deleted file mode 100644 index 88854d4..0000000 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDataService.java +++ /dev/null @@ -1,191 +0,0 @@ -package com.iemr.common.identity.service.elasticsearch; - -import java.math.BigInteger; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iemr.common.identity.domain.MBeneficiarymapping; -import com.iemr.common.identity.dto.AbhaAddressDTO; -import com.iemr.common.identity.dto.BenDetailDTO; -import com.iemr.common.identity.dto.BeneficiariesDTO; -import com.iemr.common.identity.repo.BenMappingRepo; -import com.iemr.common.identity.repo.V_BenAdvanceSearchRepo; - -/** - * Service to fetch beneficiary data directly from database - * Used for Elasticsearch sync to avoid circular dependencies - */ -@Service -public class BeneficiaryDataService { - - private static final Logger logger = LoggerFactory.getLogger(BeneficiaryDataService.class); - - @Autowired - private BenMappingRepo mappingRepo; - - @Autowired - private V_BenAdvanceSearchRepo v_BenAdvanceSearchRepo; - - - /** - * Fetch beneficiary data directly from database by benRegId - * This bypasses any Elasticsearch caching to get fresh database data - * - * @param benRegId - * @return - */ - public BeneficiariesDTO getBeneficiaryFromDatabase(BigInteger benRegId) { - int maxRetries = 3; - int retryCount = 0; - - while (retryCount < maxRetries) { - try { - logger.debug("Fetching beneficiary from database: benRegId={}, attempt={}", benRegId, retryCount + 1); - - MBeneficiarymapping mapping = mappingRepo.findByBenRegIdWithDetails(benRegId); - - if (mapping == null) { - logger.warn("Beneficiary mapping not found: benRegId={}", benRegId); - return null; - } - - BeneficiariesDTO dto = convertToDTO(mapping); - - logger.debug("Successfully fetched beneficiary: benRegId={}", benRegId); - return dto; - - } catch (org.springframework.orm.jpa.JpaSystemException e) { - retryCount++; - logger.warn("Database connection error for benRegId={}, attempt {}/{}: {}", - benRegId, retryCount, maxRetries, e.getMessage()); - - if (retryCount >= maxRetries) { - logger.error("Max retries reached for benRegId={}", benRegId); - return null; - } - - try { - Thread.sleep(1000 * retryCount); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - return null; - } - - } catch (Exception e) { - logger.error("Error fetching beneficiary from database: benRegId={}, error={}", - benRegId, e.getMessage(), e); - return null; - } - } - - return null; - } - - /** - * Convert MBeneficiarymapping entity to BeneficiariesDTO - */ - private BeneficiariesDTO convertToDTO(MBeneficiarymapping mapping) { - BeneficiariesDTO dto = new BeneficiariesDTO(); - - try { - dto.setBenRegId(mapping.getBenRegId()); - dto.setBenMapId(mapping.getBenMapId()); - - if (mapping.getMBeneficiaryregidmapping() != null) { - dto.setBenId(mapping.getMBeneficiaryregidmapping().getBeneficiaryID()); - } - - if (mapping.getMBeneficiarycontact() != null) { - dto.setPreferredPhoneNum(mapping.getMBeneficiarycontact().getPreferredPhoneNum()); - } - - if (mapping.getMBeneficiarydetail() != null) { - BenDetailDTO detailDTO = new BenDetailDTO(); - - detailDTO.setFirstName(mapping.getMBeneficiarydetail().getFirstName()); - detailDTO.setLastName(mapping.getMBeneficiarydetail().getLastName()); - - if (mapping.getMBeneficiarydetail().getDob() != null) { - detailDTO.setBeneficiaryAge(calculateAge(mapping.getMBeneficiarydetail().getDob())); - } - - dto.setBeneficiaryDetails(detailDTO); - } - try { - logger.info("Fetching ABHA details for benRegId={}", dto.getBenRegId()); - if (dto.getBenRegId() != null) { - List abhaList = v_BenAdvanceSearchRepo.getBenAbhaDetailsByBenRegID(dto.getBenRegId()); - - if (abhaList != null && !abhaList.isEmpty()) { - List abhaDTOList = new java.util.ArrayList<>(); - - for (Object[] objArr : abhaList) { - AbhaAddressDTO abhaDTO = new AbhaAddressDTO(); - abhaDTO.setBeneficiaryRegID(dto.getBenRegId()); - - // objArr[1] -> HealthID (ABHA Address) - if (objArr[1] != null) { - abhaDTO.setHealthID(objArr[1].toString()); - } - - // objArr[2] -> HealthIDNumber (ABHA Number) - if (objArr[2] != null) { - abhaDTO.setHealthIDNumber(objArr[2].toString()); - } - - // objArr[3] -> Authentication Mode - if (objArr[3] != null) { - abhaDTO.setAuthenticationMode(objArr[3].toString()); - } - - // objArr[4] -> Created Date - if (objArr[4] != null) { - abhaDTO.setCreatedDate((java.sql.Timestamp) objArr[4]); - } - - abhaDTOList.add(abhaDTO); - } - - dto.setAbhaDetails(abhaDTOList); - logger.info("ABHA details fetched: count={}", abhaDTOList.size()); - } - } -} catch (Exception e) { - logger.error("Error while fetching ABHA details for benRegId={}", - dto.getBenRegId(), e); -} - - - logger.debug("Successfully converted mapping to DTO: benRegId={}", mapping.getBenRegId()); - - } catch (Exception e) { - logger.error("Error converting mapping to DTO: {}", e.getMessage(), e); - } - - return dto; - } - - /** - * Calculate age from date of birth - */ - private Integer calculateAge(java.sql.Timestamp dob) { - try { - if (dob == null) { - return null; - } - - java.time.LocalDate birthDate = dob.toLocalDateTime().toLocalDate(); - java.time.LocalDate now = java.time.LocalDate.now(); - - return java.time.Period.between(birthDate, now).getYears(); - - } catch (Exception e) { - logger.error("Error calculating age: {}", e.getMessage()); - return null; - } - } -} \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java index 763890a..cdca328 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java @@ -48,7 +48,6 @@ public List getBeneficiariesBatch(List benRegId List results = mappingRepo.findCompleteDataByBenRegIds(benRegIds); logger.info("Fetched {} complete beneficiary records", results.size()); - // Batch fetch ABHA details for ALL beneficiaries at once Map abhaMap = batchFetchAbhaData(benRegIds); @@ -149,13 +148,12 @@ private Map batchFetchAbhaDetails(List benRegIds) { abhaData.setHealthID(record[1] != null ? record[1].toString() : null); abhaData.setHealthIDNumber(record[2] != null ? record[2].toString() : null); abhaData.setAuthenticationMode(record[3] != null ? record[3].toString() : null); - abhaData.setAbhaCreatedDate(record[4] !=null ? record[4].toString() : null); - + abhaData.setAbhaCreatedDate(record[4] != null ? record[4].toString() : null); + abhaMap.put(benRegId, abhaData); } } catch (Exception e) { logger.error("Error processing ABHA record: {}", e.getMessage()); - // Continue processing other records } } @@ -181,38 +179,34 @@ public BeneficiaryDocument getBeneficiaryFromDatabase(BigInteger benRegId) { } /** - * Enrich document with ABHA details from m_benhealthidmapping table + * Fetch single beneficiary WITH fresh ABHA data + * Use this for real-time sync (create/update operations) */ - private void enrichWithAbhaDetails(BeneficiaryDocument doc) { - try { - if (doc.getBenRegId() == null) { - return; - } - - BigInteger benRegId = BigInteger.valueOf(doc.getBenRegId()); - List abhaList = v_BenAdvanceSearchRepo.getBenAbhaDetailsByBenRegID(benRegId); - - if (abhaList != null && !abhaList.isEmpty()) { - // Get the first (most recent) ABHA record - Object[] abhaRecord = abhaList.get(0); + @Transactional(readOnly = true, timeout = 10) + public BeneficiaryDocument getBeneficiaryWithAbhaDetails(BigInteger benRegId) { + if (benRegId == null) { + return null; + } + // Fetch beneficiary + ABHA in one call + List ids = List.of(benRegId); + List results = getBeneficiariesBatch(ids); - // objArr[1] -> HealthID (ABHA Address) - if (abhaRecord[1] != null) { - doc.setHealthID(abhaRecord[1].toString()); - } + if (results.isEmpty()) { + logger.warn("No beneficiary found for benRegId={}", benRegId); + return null; + } - // objArr[2] -> HealthIDNumber (ABHA Number) - if (abhaRecord[2] != null) { - doc.setAbhaID(abhaRecord[2].toString()); - } + BeneficiaryDocument doc = results.get(0); - logger.info("ABHA details enriched for benRegId={}, healthID={}, abhaID={}", - benRegId, doc.getHealthID(), doc.getAbhaID()); - } - } catch (Exception e) { - logger.error("Error enriching ABHA details for benRegId={}: {}", - doc.getBenRegId(), e.getMessage()); + // Log for debugging + if (doc.getHealthID() != null || doc.getAbhaID() != null) { + logger.info("Beneficiary has ABHA: benRegId={}, healthID={}, abhaID={}", + benRegId, doc.getHealthID(), doc.getAbhaID()); + } else { + logger.debug("No ABHA details for benRegId={}", benRegId); } + + return doc; } /** @@ -370,8 +364,8 @@ public String getAuthenticationMode() { public void setAuthenticationMode(String authenticationMode) { this.authenticationMode = authenticationMode; } - - public String getAbhaCreatedDate() { + + public String getAbhaCreatedDate() { return abhaCreatedDate; } diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java index 82d6858..0e8bff1 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java @@ -1,6 +1,7 @@ package com.iemr.common.identity.service.elasticsearch; import java.math.BigInteger; +import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +11,7 @@ import org.springframework.stereotype.Service; import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch.core.DeleteRequest; import co.elastic.clients.elasticsearch.core.IndexRequest; @@ -41,7 +43,7 @@ public class BeneficiaryElasticsearchIndexUpdater { * Called after beneficiary is created/updated in database */ @Async("elasticsearchSyncExecutor") - public void syncBeneficiaryAsync(BigInteger benRegId) { + public void syncBeneficiaryAsyncOld(BigInteger benRegId) { if (!esEnabled) { logger.debug("Elasticsearch is disabled, skipping sync"); return; @@ -63,15 +65,14 @@ public void syncBeneficiaryAsync(BigInteger benRegId) { } IndexRequest request = IndexRequest.of(i -> i - .index(beneficiaryIndex) - .id(doc.getBenId()) - .document(doc) - ); + .index(beneficiaryIndex) + .id(doc.getBenId()) + .document(doc)); esClient.index(request); - logger.info("Successfully synced beneficiary to Elasticsearch: benRegId={}, benId={}", - benRegId, doc.getBenId()); + logger.info("Successfully synced beneficiary to Elasticsearch: benRegId={}, benId={}", + benRegId, doc.getBenId()); } catch (Exception e) { logger.error("Error syncing beneficiary {} to Elasticsearch: {}", benRegId, e.getMessage(), e); @@ -92,9 +93,8 @@ public void deleteBeneficiaryAsync(String benId) { logger.info("Starting async delete for benId: {}", benId); DeleteRequest request = DeleteRequest.of(d -> d - .index(beneficiaryIndex) - .id(benId) - ); + .index(beneficiaryIndex) + .id(benId)); esClient.delete(request); @@ -104,4 +104,31 @@ public void deleteBeneficiaryAsync(String benId) { logger.error("Error deleting beneficiary {} from Elasticsearch: {}", benId, e.getMessage(), e); } } + + @Async + public CompletableFuture syncBeneficiaryAsync(BigInteger benRegId) { + BeneficiaryDocument document = dataService.getBeneficiaryWithAbhaDetails(benRegId); + if (document == null) { + logger.warn("No data found for benRegId: {}", benRegId); + return CompletableFuture.completedFuture(null); + } + + // Log ABHA for verification + logger.info("Syncing benRegId={} with ABHA: healthID={}, abhaID={}", + benRegId, document.getHealthID(), document.getAbhaID()); + + try{ + // Index to ES + esClient.index(i -> i + .index(beneficiaryIndex) + .id(String.valueOf(benRegId)) + .document(document).refresh(Refresh.WaitFor)); + + logger.info("Successfully synced benRegId: {} to ES", benRegId); + } catch (Exception e) { + logger.error("Error syncing beneficiary {} to Elasticsearch: {}", benRegId, e.getMessage(), e); + } + return CompletableFuture.completedFuture(null); + + } } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java deleted file mode 100644 index 0b205fe..0000000 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchMappingUpdateService.java +++ /dev/null @@ -1,178 +0,0 @@ -package com.iemr.common.identity.service.elasticsearch; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.mapping.Property; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.HashMap; -import java.util.Map; - -/** - * Service to update Elasticsearch mappings without deleting the index - * Use this to add new fields to existing index - */ -@Service -public class ElasticsearchMappingUpdateService { - - private static final Logger logger = LoggerFactory.getLogger(ElasticsearchMappingUpdateService.class); - - @Autowired - private ElasticsearchClient esClient; - - @Value("${elasticsearch.index.beneficiary}") - private String beneficiaryIndex; - - /** - * Add ABHA-related fields to existing index mapping - * This does NOT require deleting the index or re-syncing data - * - * NOTE: You can only ADD new fields, you cannot CHANGE existing field types - */ - public void addAbhaFieldsToExistingIndex() throws Exception { - logger.info("Adding ABHA fields to existing index: {}", beneficiaryIndex); - - try { - // Check if index exists - boolean indexExists = esClient.indices().exists(e -> e.index(beneficiaryIndex)).value(); - - if (!indexExists) { - logger.error("Index {} does not exist. Create it first.", beneficiaryIndex); - throw new RuntimeException("Index does not exist: " + beneficiaryIndex); - } - - // Update mapping by adding new properties - esClient.indices().putMapping(pm -> pm - .index(beneficiaryIndex) - .properties("healthID", Property.of(p -> p.keyword(k -> k))) - .properties("healthIDNumber", Property.of(p -> p.keyword(k -> k))) - .properties("abhaID", Property.of(p -> p.keyword(k -> k))) - .properties("abhaCreatedDate", Property.of(p -> p.keyword(k -> k))) - .properties("abhaAuthMode", Property.of(p -> p.keyword(k -> k))) - - // Nested object for multiple ABHA addresses - .properties("abhaAddresses", Property.of(p -> p.nested(n -> n - .properties("healthID", Property.of(fp -> fp.keyword(k -> k))) - .properties("healthIDNumber", Property.of(fp -> fp.keyword(k -> k))) - .properties("authenticationMode", Property.of(fp -> fp.keyword(k -> k))) - .properties("createdDate", Property.of(fp -> fp.date(d -> d))) - ))) - - // Add villageID and villageName if missing - .properties("villageID", Property.of(p -> p.integer(i -> i))) - .properties("villageName", Property.of(p -> p.keyword(k -> k))) - ); - - logger.info("ABHA fields added successfully to index: {}", beneficiaryIndex); - logger.info("No data loss - existing documents are intact"); - logger.info("New fields will be populated when documents are updated/synced"); - - // Verify the update - verifyMappingUpdate(); - - } catch (Exception e) { - logger.error("Failed to add ABHA fields to index", e); - throw e; - } - } - - /** - * Verify that new fields were added successfully - */ - private void verifyMappingUpdate() throws Exception { - var mapping = esClient.indices().getMapping(g -> g.index(beneficiaryIndex)); - - logger.info("Current index mapping properties:"); - mapping.get(beneficiaryIndex).mappings().properties().keySet().forEach(field -> { - logger.info(" - {}", field); - }); - } - - /** - * Update existing documents with ABHA data (incremental sync) - * This updates only documents that have ABHA details in the database - * - * Call this after adding new fields to populate them with existing data - */ - public Map syncAbhaDataToExistingDocuments() { - logger.info("Starting incremental ABHA data sync to existing ES documents"); - - Map result = new HashMap<>(); - int successCount = 0; - int failureCount = 0; - - try { - // This will be handled by your existing BeneficiaryElasticsearchIndexUpdater - // which already fetches ABHA details from v_BenAdvanceSearchRepo - - // You can either: - // 1. Sync all beneficiaries (slow but complete) - // 2. Sync only beneficiaries with ABHA data (fast and targeted) - - logger.info("Use the existing syncAllBeneficiaries() method"); - logger.info("It will automatically populate new ABHA fields"); - - result.put("message", 0); - result.put("recommendation", 1); - - } catch (Exception e) { - logger.error("Error during ABHA data sync", e); - } - - result.put("success", successCount); - result.put("failed", failureCount); - return result; - } - - /** - * Get current index statistics and mapping info - */ - public Map getIndexInfo() throws Exception { - Map info = new HashMap<>(); - - // Get mapping - var mapping = esClient.indices().getMapping(g -> g.index(beneficiaryIndex)); - info.put("totalFields", mapping.get(beneficiaryIndex).mappings().properties().size()); - - // Get stats - var stats = esClient.indices().stats(s -> s.index(beneficiaryIndex)); - info.put("documentCount", stats.indices().get(beneficiaryIndex).primaries().docs().count()); - info.put("sizeInBytes", stats.indices().get(beneficiaryIndex).primaries().store().sizeInBytes()); - - // Get settings - var settings = esClient.indices().getSettings(g -> g.index(beneficiaryIndex)); - info.put("numberOfShards", settings.get(beneficiaryIndex).settings().index().numberOfShards()); - info.put("numberOfReplicas", settings.get(beneficiaryIndex).settings().index().numberOfReplicas()); - info.put("refreshInterval", settings.get(beneficiaryIndex).settings().index().refreshInterval().time()); - - // Check if ABHA fields exist - boolean hasAbhaFields = mapping.get(beneficiaryIndex).mappings().properties() - .containsKey("healthID") && mapping.get(beneficiaryIndex).mappings().properties() - .containsKey("abhaID"); - info.put("hasAbhaFields", hasAbhaFields); - - return info; - } - - /** - * Add any additional fields to index without deleting - * Use this template for future field additions - */ - public void addCustomFieldsToIndex(Map newFields) throws Exception { - logger.info("Adding {} custom fields to index: {}", newFields.size(), beneficiaryIndex); - - esClient.indices().putMapping(pm -> { - var builder = pm.index(beneficiaryIndex); - newFields.forEach((fieldName, property) -> { - builder.properties(fieldName, property); - logger.info(" Adding field: {}", fieldName); - }); - return builder; - }); - - logger.info("Custom fields added successfully"); - } -} \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index 14f9122..b65f435 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -18,6 +18,7 @@ import java.math.BigDecimal; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -169,7 +170,8 @@ public List> universalSearch(String query, Integer userId) { .filter(f -> f .includes("benRegId", "beneficiaryID", "firstName", "lastName", "genderID", "genderName", "dOB", "phoneNum", - "stateID", "stateName", "districtID", "blockID", "villageID", "healthID", "abhaID", + "stateID", "stateName", "districtID", "blockID", "villageID", "healthID", + "abhaID", "abhaCreatedDate", "familyID", "fatherName", "spouseName", "age", "createdBy", "createdDate", @@ -510,166 +512,181 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat Map result = new HashMap<>(); try { - // Basic fields from ES - result.put("beneficiaryRegID", esData.getBenRegId()); - result.put("beneficiaryID", esData.getBeneficiaryID()); - result.put("firstName", esData.getFirstName()); - result.put("lastName", esData.getLastName() != null ? esData.getLastName() : ""); - result.put("genderID", esData.getGenderID()); - result.put("genderName", esData.getGenderName()); - result.put("dob", esData.getDOB()); - result.put("dOB", esData.getDOB()); - result.put("age", esData.getAge()); - result.put("actualAge", esData.getAge()); // NEW - result.put("ageUnits", "Years"); // NEW - - result.put("fatherName", esData.getFatherName() != null ? esData.getFatherName() : ""); - result.put("spouseName", esData.getSpouseName() != null ? esData.getSpouseName() : ""); - result.put("isHIVPos", esData.getIsHIVPos() != null ? esData.getIsHIVPos() : ""); - - result.put("createdBy", esData.getCreatedBy()); - result.put("createdDate", esData.getCreatedDate()); - result.put("lastModDate", esData.getLastModDate()); - result.put("benAccountID", esData.getBenAccountID()); - - // Family ID - use both formats - result.put("familyID", esData.getFamilyID()); - result.put("familyId", esData.getFamilyID()); // NEW - alternate key - - // Marital status - NEW - result.put("maritalStatusName", null); // TODO: Add to ES if available - result.put("maritalStatus", new HashMap<>()); - - // Head of family - NEW (add these fields to BeneficiariesESDTO if available) - result.put("headOfFamily_RelationID", null); - result.put("headOfFamily_Relation", null); - - // ABHA Details - List> abhaDetails = new ArrayList<>(); - if (esData.getHealthID() != null || esData.getAbhaID() != null) { - Map abhaDetail = new HashMap<>(); - abhaDetail.put("healthIDNumber", esData.getAbhaID()); - abhaDetail.put("healthID", esData.getAbhaID()); - if (esData.getAbhaCreatedDate() != null) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S"); - LocalDateTime localDateTime = LocalDateTime.parse(esData.getAbhaCreatedDate(), formatter); - long createdDateMillis = localDateTime - .atZone(ZoneId.of("Asia/Kolkata")) - .toInstant() - .toEpochMilli(); - abhaDetail.put("createdDate", createdDateMillis); + // Basic fields from ES + result.put("beneficiaryRegID", esData.getBenRegId()); + result.put("beneficiaryID", esData.getBeneficiaryID()); + result.put("firstName", esData.getFirstName()); + result.put("lastName", esData.getLastName() != null ? esData.getLastName() : ""); + result.put("genderID", esData.getGenderID()); + result.put("genderName", esData.getGenderName()); + result.put("dob", esData.getDOB()); + result.put("dOB", esData.getDOB()); + result.put("age", esData.getAge()); + result.put("actualAge", esData.getAge()); // NEW + result.put("ageUnits", "Years"); // NEW + + result.put("fatherName", esData.getFatherName() != null ? esData.getFatherName() : ""); + result.put("spouseName", esData.getSpouseName() != null ? esData.getSpouseName() : ""); + result.put("isHIVPos", esData.getIsHIVPos() != null ? esData.getIsHIVPos() : ""); + + result.put("createdBy", esData.getCreatedBy()); + result.put("createdDate", esData.getCreatedDate()); + result.put("lastModDate", esData.getLastModDate()); + result.put("benAccountID", esData.getBenAccountID()); + + // Family ID - use both formats + result.put("familyID", esData.getFamilyID()); + result.put("familyId", esData.getFamilyID()); // NEW - alternate key + + // Marital status - NEW + result.put("maritalStatusName", null); // TODO: Add to ES if available + result.put("maritalStatus", new HashMap<>()); + + // Head of family - NEW (add these fields to BeneficiariesESDTO if available) + result.put("headOfFamily_RelationID", null); + result.put("headOfFamily_Relation", null); + + // ABHA Details + List> abhaDetails = new ArrayList<>(); + if (esData.getHealthID() != null || esData.getAbhaID() != null) { + Map abhaDetail = new HashMap<>(); + abhaDetail.put("healthIDNumber", esData.getAbhaID()); + abhaDetail.put("healthID", esData.getAbhaID()); + if (esData.getAbhaCreatedDate() != null) { + + String dateStr = esData.getAbhaCreatedDate(); + ZoneId zoneId = ZoneId.of("Asia/Kolkata"); + long createdDateMillis; + + if (dateStr.length() == 10) { + // yyyy-MM-dd + createdDateMillis = LocalDate.parse(dateStr) + .atStartOfDay(zoneId) + .toInstant() + .toEpochMilli(); + } else { + // yyyy-MM-dd HH:mm:ss OR yyyy-MM-dd HH:mm:ss.S + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss[.S]"); + + createdDateMillis = LocalDateTime.parse(dateStr, formatter) + .atZone(zoneId) + .toInstant() + .toEpochMilli(); + } + + abhaDetail.put("createdDate", createdDateMillis); + } + abhaDetail.put("beneficiaryRegID", esData.getBenRegId()); + abhaDetails.add(abhaDetail); } - abhaDetail.put("beneficiaryRegID", esData.getBenRegId()); - abhaDetails.add(abhaDetail); - } - result.put("abhaDetails", abhaDetails); - - // Gender object - Map mGender = new HashMap<>(); - mGender.put("genderID", esData.getGenderID()); - mGender.put("genderName", esData.getGenderName()); - result.put("m_gender", mGender); - - // Demographics - UPDATED with all fields - Map demographics = new HashMap<>(); - demographics.put("beneficiaryRegID", esData.getBenRegId()); - - // Current address - demographics.put("stateID", esData.getStateID()); - demographics.put("stateName", esData.getStateName()); - demographics.put("districtID", esData.getDistrictID()); - demographics.put("districtName", esData.getDistrictName()); - demographics.put("blockID", esData.getBlockID()); - demographics.put("blockName", esData.getBlockName()); - demographics.put("villageID", esData.getVillageID()); // FIXED - demographics.put("villageName", esData.getVillageName()); // FIXED - - demographics.put("districtBranchID", esData.getVillageID()); - demographics.put("districtBranchName", esData.getVillageName()); - demographics.put("parkingPlaceID", esData.getParkingPlaceID()); - demographics.put("servicePointID", esData.getServicePointID()); - demographics.put("servicePointName", esData.getServicePointName()); - demographics.put("createdBy", esData.getCreatedBy()); - - // State object - Map mState = new HashMap<>(); - mState.put("stateID", esData.getStateID()); - mState.put("stateName", esData.getStateName()); - mState.put("stateCode", null); - mState.put("countryID", 1); - demographics.put("m_state", mState); - - // District object - Map mDistrict = new HashMap<>(); - mDistrict.put("districtID", esData.getDistrictID()); - mDistrict.put("districtName", esData.getDistrictName()); - mDistrict.put("stateID", esData.getStateID()); - demographics.put("m_district", mDistrict); - - // Block object - Map mBlock = new HashMap<>(); - mBlock.put("blockID", esData.getBlockID()); - mBlock.put("blockName", esData.getBlockName()); - mBlock.put("districtID", esData.getDistrictID()); - mBlock.put("stateID", esData.getStateID()); - demographics.put("m_districtblock", mBlock); - - // Branch mapping object - Map mBranch = new HashMap<>(); - mBranch.put("districtBranchID", null); - mBranch.put("blockID", esData.getBlockID()); - mBranch.put("villageName", esData.getVillageName()); // FIXED - mBranch.put("pinCode", esData.getPinCode()); - demographics.put("m_districtbranchmapping", mBranch); - - result.put("i_bendemographics", demographics); - - // Phone maps - List> benPhoneMaps = new ArrayList<>(); - if (esData.getPhoneNum() != null && !esData.getPhoneNum().isEmpty()) { - Map phoneMap = new HashMap<>(); - phoneMap.put("benPhMapID", 1L); - phoneMap.put("benificiaryRegID", esData.getBenRegId()); - phoneMap.put("parentBenRegID", esData.getBenRegId()); - phoneMap.put("benRelationshipID", 1); - phoneMap.put("phoneNo", esData.getPhoneNum()); - - Map relationType = new HashMap<>(); - relationType.put("benRelationshipID", 1); - relationType.put("benRelationshipType", "Self"); - phoneMap.put("benRelationshipType", relationType); - - benPhoneMaps.add(phoneMap); + result.put("abhaDetails", abhaDetails); + + // Gender object + Map mGender = new HashMap<>(); + mGender.put("genderID", esData.getGenderID()); + mGender.put("genderName", esData.getGenderName()); + result.put("m_gender", mGender); + + // Demographics - UPDATED with all fields + Map demographics = new HashMap<>(); + demographics.put("beneficiaryRegID", esData.getBenRegId()); + + // Current address + demographics.put("stateID", esData.getStateID()); + demographics.put("stateName", esData.getStateName()); + demographics.put("districtID", esData.getDistrictID()); + demographics.put("districtName", esData.getDistrictName()); + demographics.put("blockID", esData.getBlockID()); + demographics.put("blockName", esData.getBlockName()); + demographics.put("villageID", esData.getVillageID()); // FIXED + demographics.put("villageName", esData.getVillageName()); // FIXED + + demographics.put("districtBranchID", esData.getVillageID()); + demographics.put("districtBranchName", esData.getVillageName()); + demographics.put("parkingPlaceID", esData.getParkingPlaceID()); + demographics.put("servicePointID", esData.getServicePointID()); + demographics.put("servicePointName", esData.getServicePointName()); + demographics.put("createdBy", esData.getCreatedBy()); + + // State object + Map mState = new HashMap<>(); + mState.put("stateID", esData.getStateID()); + mState.put("stateName", esData.getStateName()); + mState.put("stateCode", null); + mState.put("countryID", 1); + demographics.put("m_state", mState); + + // District object + Map mDistrict = new HashMap<>(); + mDistrict.put("districtID", esData.getDistrictID()); + mDistrict.put("districtName", esData.getDistrictName()); + mDistrict.put("stateID", esData.getStateID()); + demographics.put("m_district", mDistrict); + + // Block object + Map mBlock = new HashMap<>(); + mBlock.put("blockID", esData.getBlockID()); + mBlock.put("blockName", esData.getBlockName()); + mBlock.put("districtID", esData.getDistrictID()); + mBlock.put("stateID", esData.getStateID()); + demographics.put("m_districtblock", mBlock); + + // Branch mapping object + Map mBranch = new HashMap<>(); + mBranch.put("districtBranchID", null); + mBranch.put("blockID", esData.getBlockID()); + mBranch.put("villageName", esData.getVillageName()); // FIXED + mBranch.put("pinCode", esData.getPinCode()); + demographics.put("m_districtbranchmapping", mBranch); + + result.put("i_bendemographics", demographics); + + // Phone maps + List> benPhoneMaps = new ArrayList<>(); + if (esData.getPhoneNum() != null && !esData.getPhoneNum().isEmpty()) { + Map phoneMap = new HashMap<>(); + phoneMap.put("benPhMapID", 1L); + phoneMap.put("benificiaryRegID", esData.getBenRegId()); + phoneMap.put("parentBenRegID", esData.getBenRegId()); + phoneMap.put("benRelationshipID", 1); + phoneMap.put("phoneNo", esData.getPhoneNum()); + + Map relationType = new HashMap<>(); + relationType.put("benRelationshipID", 1); + relationType.put("benRelationshipType", "Self"); + phoneMap.put("benRelationshipType", relationType); + + benPhoneMaps.add(phoneMap); + } + result.put("benPhoneMaps", benPhoneMaps); + + // Boolean flags + result.put("isConsent", false); + result.put("changeInSelfDetails", false); + result.put("changeInAddress", false); + result.put("changeInContacts", false); + result.put("changeInIdentities", false); + result.put("changeInOtherDetails", false); + result.put("changeInFamilyDetails", false); + result.put("changeInAssociations", false); + result.put("changeInBankDetails", false); + result.put("changeInBenImage", false); + result.put("is1097", false); + result.put("emergencyRegistration", false); + result.put("passToNurse", false); + + // Empty objects/arrays + result.put("m_title", new HashMap<>()); + result.put("maritalStatus", new HashMap<>()); + result.put("beneficiaryIdentities", new ArrayList<>()); + + } catch (Exception e) { + logger.error("Error mapping ES result: {}", e.getMessage(), e); + return null; } - result.put("benPhoneMaps", benPhoneMaps); - - // Boolean flags - result.put("isConsent", false); - result.put("changeInSelfDetails", false); - result.put("changeInAddress", false); - result.put("changeInContacts", false); - result.put("changeInIdentities", false); - result.put("changeInOtherDetails", false); - result.put("changeInFamilyDetails", false); - result.put("changeInAssociations", false); - result.put("changeInBankDetails", false); - result.put("changeInBenImage", false); - result.put("is1097", false); - result.put("emergencyRegistration", false); - result.put("passToNurse", false); - - // Empty objects/arrays - result.put("m_title", new HashMap<>()); - result.put("maritalStatus", new HashMap<>()); - result.put("beneficiaryIdentities", new ArrayList<>()); - - } catch (Exception e) { - logger.error("Error mapping ES result: {}", e.getMessage(), e); - return null; - } - return result; -} + return result; + } /** * Direct database search as fallback @@ -691,141 +708,142 @@ private List> searchInDatabaseDirectly(String query) { /** * Map database result to expected API format */ - private Map mapToExpectedFormat(Object[] row) { - Map result = new HashMap<>(); - - try { - Long beneficiaryRegID = getLong(row[0]); - String beneficiaryID = getString(row[1]); - String firstName = getString(row[2]); - String lastName = getString(row[3]); - Integer genderID = getInteger(row[4]); - String genderName = getString(row[5]); - Date dob = getDate(row[6]); - Integer age = getInteger(row[7]); - String fatherName = getString(row[8]); - String spouseName = getString(row[9]); - String isHIVPos = getString(row[10]); - String createdBy = getString(row[11]); - Date createdDate = getDate(row[12]); - Long lastModDate = getLong(row[13]); - Long benAccountID = getLong(row[14]); - - Integer stateID = getInteger(row[15]); - String stateName = getString(row[16]); - Integer districtID = getInteger(row[17]); - String districtName = getString(row[18]); - Integer blockID = getInteger(row[19]); - String blockName = getString(row[20]); - String pinCode = getString(row[21]); - Integer servicePointID = getInteger(row[22]); - String servicePointName = getString(row[23]); - Integer parkingPlaceID = getInteger(row[24]); - String phoneNum = getString(row[25]); - - Integer villageID = getInteger(row[26]); - String villageName = getString(row[27]); - - result.put("beneficiaryRegID", beneficiaryRegID); - result.put("beneficiaryID", beneficiaryID); - result.put("firstName", firstName); - result.put("lastName", lastName != null ? lastName : ""); - result.put("genderID", genderID); - result.put("genderName", genderName); - result.put("dOB", dob); - result.put("dob", dob); - result.put("age", age); - result.put("actualAge", age); // NEW - result.put("ageUnits", "Years"); // NEW - result.put("fatherName", fatherName != null ? fatherName : ""); - result.put("spouseName", spouseName != null ? spouseName : ""); - result.put("isHIVPos", isHIVPos != null ? isHIVPos : ""); - result.put("createdBy", createdBy); - result.put("createdDate", createdDate); - result.put("lastModDate", lastModDate); - result.put("benAccountID", benAccountID); - - // NEW fields - result.put("maritalStatusName", null); - result.put("maritalStatus", new HashMap<>()); - - Map mGender = new HashMap<>(); - mGender.put("genderID", genderID); - mGender.put("genderName", genderName); - result.put("m_gender", mGender); - - Map demographics = new HashMap<>(); - demographics.put("beneficiaryRegID", beneficiaryRegID); - demographics.put("stateID", stateID); - demographics.put("stateName", stateName); - demographics.put("districtID", districtID); - demographics.put("districtName", districtName); - demographics.put("blockID", blockID); - demographics.put("blockName", blockName); - demographics.put("villageID", villageID); - demographics.put("villageName", villageName); - demographics.put("districtBranchID", villageID); - demographics.put("districtBranchName", villageName); - demographics.put("parkingPlaceID", parkingPlaceID); - demographics.put("servicePointID", servicePointID); - demographics.put("servicePointName", servicePointName); - demographics.put("createdBy", createdBy); - - Map mState = new HashMap<>(); - mState.put("stateID", stateID); - mState.put("stateName", stateName); - mState.put("stateCode", null); - mState.put("countryID", 1); - demographics.put("m_state", mState); - - Map mDistrict = new HashMap<>(); - mDistrict.put("districtID", districtID); - mDistrict.put("districtName", districtName); - mDistrict.put("stateID", stateID); - demographics.put("m_district", mDistrict); - - Map mBlock = new HashMap<>(); - mBlock.put("blockID", blockID); - mBlock.put("blockName", blockName); - mBlock.put("districtID", districtID); - mBlock.put("stateID", stateID); - demographics.put("m_districtblock", mBlock); - - Map mBranch = new HashMap<>(); - mBranch.put("districtBranchID", null); - mBranch.put("blockID", blockID); - mBranch.put("villageName", villageName); // FIXED - mBranch.put("pinCode", pinCode); - demographics.put("m_districtbranchmapping", mBranch); - - result.put("i_bendemographics", demographics); - - List> benPhoneMaps = fetchPhoneNumbers(beneficiaryRegID); - result.put("benPhoneMaps", benPhoneMaps); - - result.put("isConsent", false); - result.put("m_title", new HashMap<>()); - result.put("maritalStatus", new HashMap<>()); - result.put("changeInSelfDetails", false); - result.put("changeInAddress", false); - result.put("changeInContacts", false); - result.put("changeInIdentities", false); - result.put("changeInOtherDetails", false); - result.put("changeInFamilyDetails", false); - result.put("changeInAssociations", false); - result.put("changeInBankDetails", false); - result.put("changeInBenImage", false); - result.put("is1097", false); - result.put("emergencyRegistration", false); - result.put("passToNurse", false); - result.put("beneficiaryIdentities", new ArrayList<>()); - - } catch (Exception e) { - logger.error("Error mapping result: {}", e.getMessage(), e); + private Map mapToExpectedFormat(Object[] row) { + Map result = new HashMap<>(); + + try { + Long beneficiaryRegID = getLong(row[0]); + String beneficiaryID = getString(row[1]); + String firstName = getString(row[2]); + String lastName = getString(row[3]); + Integer genderID = getInteger(row[4]); + String genderName = getString(row[5]); + Date dob = getDate(row[6]); + Integer age = getInteger(row[7]); + String fatherName = getString(row[8]); + String spouseName = getString(row[9]); + String isHIVPos = getString(row[10]); + String createdBy = getString(row[11]); + Date createdDate = getDate(row[12]); + Long lastModDate = getLong(row[13]); + Long benAccountID = getLong(row[14]); + + Integer stateID = getInteger(row[15]); + String stateName = getString(row[16]); + Integer districtID = getInteger(row[17]); + String districtName = getString(row[18]); + Integer blockID = getInteger(row[19]); + String blockName = getString(row[20]); + String pinCode = getString(row[21]); + Integer servicePointID = getInteger(row[22]); + String servicePointName = getString(row[23]); + Integer parkingPlaceID = getInteger(row[24]); + String phoneNum = getString(row[25]); + + Integer villageID = getInteger(row[26]); + String villageName = getString(row[27]); + + result.put("beneficiaryRegID", beneficiaryRegID); + result.put("beneficiaryID", beneficiaryID); + result.put("firstName", firstName); + result.put("lastName", lastName != null ? lastName : ""); + result.put("genderID", genderID); + result.put("genderName", genderName); + result.put("dOB", dob); + result.put("dob", dob); + result.put("age", age); + result.put("actualAge", age); // NEW + result.put("ageUnits", "Years"); // NEW + result.put("fatherName", fatherName != null ? fatherName : ""); + result.put("spouseName", spouseName != null ? spouseName : ""); + result.put("isHIVPos", isHIVPos != null ? isHIVPos : ""); + result.put("createdBy", createdBy); + result.put("createdDate", createdDate); + result.put("lastModDate", lastModDate); + result.put("benAccountID", benAccountID); + + // NEW fields + result.put("maritalStatusName", null); + result.put("maritalStatus", new HashMap<>()); + + Map mGender = new HashMap<>(); + mGender.put("genderID", genderID); + mGender.put("genderName", genderName); + result.put("m_gender", mGender); + + Map demographics = new HashMap<>(); + demographics.put("beneficiaryRegID", beneficiaryRegID); + demographics.put("stateID", stateID); + demographics.put("stateName", stateName); + demographics.put("districtID", districtID); + demographics.put("districtName", districtName); + demographics.put("blockID", blockID); + demographics.put("blockName", blockName); + demographics.put("villageID", villageID); + demographics.put("villageName", villageName); + demographics.put("districtBranchID", villageID); + demographics.put("districtBranchName", villageName); + demographics.put("parkingPlaceID", parkingPlaceID); + demographics.put("servicePointID", servicePointID); + demographics.put("servicePointName", servicePointName); + demographics.put("createdBy", createdBy); + + Map mState = new HashMap<>(); + mState.put("stateID", stateID); + mState.put("stateName", stateName); + mState.put("stateCode", null); + mState.put("countryID", 1); + demographics.put("m_state", mState); + + Map mDistrict = new HashMap<>(); + mDistrict.put("districtID", districtID); + mDistrict.put("districtName", districtName); + mDistrict.put("stateID", stateID); + demographics.put("m_district", mDistrict); + + Map mBlock = new HashMap<>(); + mBlock.put("blockID", blockID); + mBlock.put("blockName", blockName); + mBlock.put("districtID", districtID); + mBlock.put("stateID", stateID); + demographics.put("m_districtblock", mBlock); + + Map mBranch = new HashMap<>(); + mBranch.put("districtBranchID", null); + mBranch.put("blockID", blockID); + mBranch.put("villageName", villageName); // FIXED + mBranch.put("pinCode", pinCode); + demographics.put("m_districtbranchmapping", mBranch); + + result.put("i_bendemographics", demographics); + + List> benPhoneMaps = fetchPhoneNumbers(beneficiaryRegID); + result.put("benPhoneMaps", benPhoneMaps); + + result.put("isConsent", false); + result.put("m_title", new HashMap<>()); + result.put("maritalStatus", new HashMap<>()); + result.put("changeInSelfDetails", false); + result.put("changeInAddress", false); + result.put("changeInContacts", false); + result.put("changeInIdentities", false); + result.put("changeInOtherDetails", false); + result.put("changeInFamilyDetails", false); + result.put("changeInAssociations", false); + result.put("changeInBankDetails", false); + result.put("changeInBenImage", false); + result.put("is1097", false); + result.put("emergencyRegistration", false); + result.put("passToNurse", false); + result.put("beneficiaryIdentities", new ArrayList<>()); + + } catch (Exception e) { + logger.error("Error mapping result: {}", e.getMessage(), e); + } + + return result; } - return result; -} /** * Fetch phone numbers for a beneficiary */ diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index ab9ed32..c0544c2 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; @@ -298,7 +299,7 @@ public boolean syncSingleBeneficiary(String benRegId) { esClient.index(i -> i .index(beneficiaryIndex) .id(doc.getBenId()) - .document(doc)); + .document(doc).refresh(Refresh.WaitFor)); logger.info("SUCCESS! Beneficiary {} synced to Elasticsearch with ABHA", benRegId); return true; From 899c4146987de96f505f0ee3e3af7a0d584ad6ac Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 21 Jan 2026 23:04:18 +0530 Subject: [PATCH 10/14] fix: refresh index --- .../ElasticsearchSyncController.java | 24 +++++++++---------- .../BeneficiaryElasticsearchIndexUpdater.java | 2 +- .../ElasticsearchSyncService.java | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java b/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java index b8ed204..8c00730 100644 --- a/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java +++ b/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java @@ -45,7 +45,7 @@ public class ElasticsearchSyncController { * Start async full sync (RECOMMENDED for millions of records) * Returns immediately with job ID for tracking * - * Usage: POST http://localhost:8080/elasticsearch/sync/start + * Usage: POST http://localhost:8080/elasticsearch/start */ @PostMapping("/start") public ResponseEntity> startAsyncFullSync( @@ -83,7 +83,7 @@ public ResponseEntity> startAsyncFullSync( /** * Get job status by ID * - * Usage: GET http://localhost:8080/elasticsearch/sync/status/1 + * Usage: GET http://localhost:8080/elasticsearch/status/1 */ @GetMapping("/status/{jobId}") public ResponseEntity> getAsyncJobStatus(@PathVariable Long jobId) { @@ -120,7 +120,7 @@ public ResponseEntity> getAsyncJobStatus(@PathVariable Long /** * Get all active jobs * - * Usage: GET http://localhost:8080/elasticsearch/sync/active + * Usage: GET http://localhost:8080/elasticsearch/active */ @GetMapping("/active") public ResponseEntity> getActiveJobs() { @@ -131,7 +131,7 @@ public ResponseEntity> getActiveJobs() { /** * Get recent jobs * - * Usage: GET http://localhost:8080/elasticsearch/sync/recent + * Usage: GET http://localhost:8080/elasticsearch/recent */ @GetMapping("/recent") public ResponseEntity> getRecentJobs() { @@ -142,7 +142,7 @@ public ResponseEntity> getRecentJobs() { /** * Resume a failed job * - * Usage: POST http://localhost:8080/elasticsearch/sync/resume/1 + * Usage: POST http://localhost:8080/elasticsearch/resume/1 */ @PostMapping("/resume/{jobId}") public ResponseEntity> resumeJob( @@ -173,7 +173,7 @@ public ResponseEntity> resumeJob( /** * Cancel a running job * - * Usage: POST http://localhost:8080/elasticsearch/sync/cancel/1 + * Usage: POST http://localhost:8080/elasticsearch/cancel/1 */ @PostMapping("/cancel/{jobId}") public ResponseEntity> cancelJob(@PathVariable Long jobId) { @@ -194,10 +194,10 @@ public ResponseEntity> cancelJob(@PathVariable Long jobId) { } /** - * LEGACY: Synchronous full sync (NOT recommended for large datasets) + * LEGACY: Synchronous full sync(NOT recommended for large datasets) * Use /start instead * - * Usage: POST http://localhost:8080/elasticsearch/sync/all + * Usage: POST http://localhost:8080/elasticsearch/all */ @PostMapping("/all") public ResponseEntity> syncAllBeneficiaries() { @@ -232,7 +232,7 @@ public ResponseEntity> syncAllBeneficiaries() { /** * Sync a single beneficiary by BenRegId * - * Usage: POST http://localhost:8080/elasticsearch/sync/single/123456 + * Usage: POST http://localhost:8080/elasticsearch/single/123456 */ @PostMapping("/single/{benRegId}") public ResponseEntity> syncSingleBeneficiary( @@ -268,7 +268,7 @@ public ResponseEntity> syncSingleBeneficiary( /** * Check sync status - compare DB count vs ES count * - * Usage: GET http://localhost:8080/elasticsearch/sync/status + * Usage: GET http://localhost:8080/elasticsearch/status */ @GetMapping("/status") public ResponseEntity checkSyncStatus() { @@ -289,7 +289,7 @@ public ResponseEntity checkSyncStatus() { /** * Health check endpoint * - * Usage: GET http://localhost:8080/elasticsearch/sync/health + * Usage: GET http://localhost:8080/elasticsearch/health */ @GetMapping("/health") public ResponseEntity> healthCheck() { @@ -304,7 +304,7 @@ public ResponseEntity> healthCheck() { /** * Debug endpoint to check if a beneficiary exists in database * - * Usage: GET http://localhost:8080/elasticsearch/sync/debug/check/123456 + * Usage: GET http://localhost:8080/elasticsearch/debug/check/123456 */ @GetMapping("/debug/check/{benRegId}") public ResponseEntity> checkBeneficiaryExists( diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java index 0e8bff1..a0255d8 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexUpdater.java @@ -122,7 +122,7 @@ public CompletableFuture syncBeneficiaryAsync(BigInteger benRegId) { esClient.index(i -> i .index(beneficiaryIndex) .id(String.valueOf(benRegId)) - .document(document).refresh(Refresh.WaitFor)); + .document(document).refresh(Refresh.True)); logger.info("Successfully synced benRegId: {} to ES", benRegId); } catch (Exception e) { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index c0544c2..0f80c12 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -299,7 +299,7 @@ public boolean syncSingleBeneficiary(String benRegId) { esClient.index(i -> i .index(beneficiaryIndex) .id(doc.getBenId()) - .document(doc).refresh(Refresh.WaitFor)); + .document(doc).refresh(Refresh.True)); logger.info("SUCCESS! Beneficiary {} synced to Elasticsearch with ABHA", benRegId); return true; From 35a909bcd537c2b83c73b28b35eb07696f603a60 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 21 Jan 2026 23:15:48 +0530 Subject: [PATCH 11/14] fix: remove duplicate dependency --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index 27341f1..57f7464 100644 --- a/pom.xml +++ b/pom.xml @@ -76,16 +76,6 @@ org.springframework.boot spring-boot-starter-data-elasticsearch - - co.elastic.clients - elasticsearch-java - 8.11.0 - - - - org.springframework.boot - spring-boot-starter-data-elasticsearch - co.elastic.clients elasticsearch-java From a993075b22d71881d29692cf4978b599d1e25cac Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 23 Jan 2026 12:12:23 +0530 Subject: [PATCH 12/14] fix: fuzzy search, resume, refresh api's --- .../ElasticsearchSyncController.java | 32 +++ .../common/identity/repo/BenDetailRepo.java | 6 + .../BeneficiaryElasticsearchIndexService.java | 203 +++++++++++------- .../elasticsearch/ElasticsearchService.java | 141 +++++++----- .../ElasticsearchSyncService.java | 8 +- src/main/resources/application.properties | 179 ++++++++------- 6 files changed, 359 insertions(+), 210 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java b/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java index 8c00730..52ddedb 100644 --- a/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java +++ b/src/main/java/com/iemr/common/identity/controller/elasticsearch/ElasticsearchSyncController.java @@ -3,6 +3,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; @@ -16,6 +17,9 @@ import java.util.Map; import org.springframework.http.ResponseEntity; import com.iemr.common.identity.utils.response.OutputResponse; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; + import com.iemr.common.identity.domain.MBeneficiarymapping; import com.iemr.common.identity.service.elasticsearch.ElasticsearchIndexingService; @@ -41,6 +45,12 @@ public class ElasticsearchSyncController { @Autowired private ElasticsearchIndexingService indexingService; + @Autowired + private ElasticsearchClient esClient; + + @Value("${elasticsearch.index.beneficiary}") + private String beneficiaryIndex; + /** * Start async full sync (RECOMMENDED for millions of records) * Returns immediately with job ID for tracking @@ -427,4 +437,26 @@ public ResponseEntity getIndexInfo() { return ResponseEntity.status(500).body(response); } } + + @PostMapping("/refresh") +public ResponseEntity> refreshIndex() { + Map response = new HashMap<>(); + + try { + logger.info("Manual refresh requested"); + + esClient.indices().refresh(r -> r.index(beneficiaryIndex)); + + response.put("status", "success"); + response.put("message", "Index refreshed - all data is now searchable"); + + return ResponseEntity.ok(response); + + } catch (Exception e) { + logger.error("Refresh failed: {}", e.getMessage(), e); + response.put("status", "error"); + response.put("message", "Refresh failed: " + e.getMessage()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(response); + } +} } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java b/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java index c543d34..085002b 100644 --- a/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java +++ b/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java @@ -178,6 +178,8 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "addr.CurrServicePoint, " + // 23 "addr.ParkingPlaceID, " + // 24 "contact.PreferredPhoneNum " + // 25 + "addr.CurrVillageId, " + // 26 + "addr.CurrVillage " + // 27 "FROM i_beneficiarymapping m " + "LEFT JOIN i_beneficiarydetails d ON m.BenDetailsId = d.BeneficiaryDetailsID " + "LEFT JOIN m_beneficiaryregidmapping brm ON brm.BenRegId = m.BenRegId " + @@ -218,6 +220,8 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "addr.CurrServicePoint, " + "addr.ParkingPlaceID, " + "contact.PreferredPhoneNum " + + "addr.CurrVillageId, " + + "addr.CurrVillage " + "FROM i_beneficiarymapping m " + "LEFT JOIN i_beneficiarydetails d ON m.BenDetailsId = d.BeneficiaryDetailsID " + "LEFT JOIN db_iemr.m_gender g ON d.GenderID = g.GenderID " + @@ -307,6 +311,8 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "addr.CurrServicePoint, " + // 23 "addr.ParkingPlaceID, " + // 24 "contact.PreferredPhoneNum " + // 25 + "addr.CurrVillageId, " + + "addr.CurrVillage " + "FROM i_beneficiarymapping m " + "LEFT JOIN i_beneficiarydetails d " + " ON m.BenDetailsId = d.BeneficiaryDetailsID " + diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexService.java index de22983..681cb5f 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryElasticsearchIndexService.java @@ -27,9 +27,9 @@ public class BeneficiaryElasticsearchIndexService { private static final Logger logger = LoggerFactory.getLogger(BeneficiaryElasticsearchIndexService.class); - private static final int BATCH_SIZE = 100; - private static final int ES_BULK_SIZE = 50; - private static final int STATUS_UPDATE_FREQUENCY = 10; + private static final int BATCH_SIZE = 2000; + private static final int ES_BULK_SIZE = 5000; + private static final int STATUS_UPDATE_FREQUENCY = 5; @Autowired private ElasticsearchClient esClient; @@ -54,18 +54,41 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { logger.info("Starting ASYNC full sync with COMPLETE data: jobId={}", jobId); ElasticsearchSyncJob job = syncJobRepository.findByJobId(jobId) - .orElseThrow(() -> new RuntimeException("Job not found: " + jobId)); + .orElseThrow(() -> new RuntimeException("Job not found: " + jobId)); try { - job.setStatus("RUNNING"); - job.setStartedAt(new Timestamp(System.currentTimeMillis())); - syncJobRepository.save(job); - - long totalCount = transactionalWrapper.countActiveBeneficiaries(); - job.setTotalRecords(totalCount); - syncJobRepository.save(job); + boolean isResume = false; + int offset = 0; + long processedCount = 0; + long successCount = 0; + long failureCount = 0; + + // Check if this is a resume from a previous run + if (job.getStatus().equals("RUNNING") && job.getCurrentOffset() != null && job.getCurrentOffset() > 0) { + isResume = true; + offset = job.getCurrentOffset(); + processedCount = job.getProcessedRecords() != null ? job.getProcessedRecords() : 0; + successCount = job.getSuccessCount() != null ? job.getSuccessCount() : 0; + failureCount = job.getFailureCount() != null ? job.getFailureCount() : 0; + + logger.info("RESUMING SYNC from offset {} (processed: {}, success: {}, failed: {})", + offset, processedCount, successCount, failureCount); + } else { + job.setStatus("RUNNING"); + job.setStartedAt(new Timestamp(System.currentTimeMillis())); + syncJobRepository.save(job); + } - logger.info("Total beneficiaries to sync: {}", totalCount); + long totalCount; + if (job.getTotalRecords() != null && job.getTotalRecords() > 0) { + totalCount = job.getTotalRecords(); + logger.info("Using cached total count: {}", totalCount); + } else { + totalCount = transactionalWrapper.countActiveBeneficiaries(); + job.setTotalRecords(totalCount); + syncJobRepository.save(job); + logger.info("Fetched total beneficiaries to sync: {}", totalCount); + } if (totalCount == 0) { job.setStatus("COMPLETED"); @@ -75,23 +98,22 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { return; } - int offset = job.getCurrentOffset() != null ? job.getCurrentOffset() : 0; - long processedCount = job.getProcessedRecords() != null ? job.getProcessedRecords() : 0; - long successCount = job.getSuccessCount() != null ? job.getSuccessCount() : 0; - long failureCount = job.getFailureCount() != null ? job.getFailureCount() : 0; - List esBatch = new ArrayList<>(); - int batchCounter = 0; - long startTime = System.currentTimeMillis(); + int batchCounter = offset / BATCH_SIZE; + long startTime = isResume ? job.getStartedAt().getTime() : System.currentTimeMillis(); + long lastProgressUpdate = System.currentTimeMillis(); + int consecutiveErrors = 0; + final int MAX_CONSECUTIVE_ERRORS = 5; // Process in batches while (offset < totalCount) { try { - logger.info("=== BATCH {} START: offset={} ===", batchCounter + 1, offset); - + logger.info("=== BATCH {} START: offset={}/{} ({:.1f}%) ===", + batchCounter + 1, offset, totalCount, (offset * 100.0 / totalCount)); + logger.debug("Calling getBeneficiaryIdsBatch(offset={}, limit={})", offset, BATCH_SIZE); List batchIds = transactionalWrapper.getBeneficiaryIdsBatch(offset, BATCH_SIZE); - + logger.info("Retrieved {} IDs from database", batchIds != null ? batchIds.size() : 0); if (batchIds == null || batchIds.isEmpty()) { @@ -99,17 +121,14 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { break; } - if (batchIds.size() > 0) { - logger.debug("First ID type: {}, value: {}", - batchIds.get(0)[0].getClass().getName(), - batchIds.get(0)[0]); - } + // Reset consecutive error counter on successful fetch + consecutiveErrors = 0; logger.debug("Converting {} IDs to BigInteger", batchIds.size()); List benRegIds = batchIds.stream() - .map(arr -> toBigInteger(arr[0])) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .map(arr -> toBigInteger(arr[0])) + .filter(Objects::nonNull) + .collect(Collectors.toList()); logger.info("Converted {} valid BigInteger IDs", benRegIds.size()); @@ -119,26 +138,10 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { continue; } - if (benRegIds.size() > 0) { - logger.debug("First converted BigInteger: {}", benRegIds.get(0)); - } - logger.info("Fetching complete data for {} beneficiaries...", benRegIds.size()); - - // *** CRITICAL: This returns COMPLETE BeneficiaryDocument with 38+ fields *** List documents = dataService.getBeneficiariesBatch(benRegIds); - + logger.info("✓ Fetched {} complete documents for batch at offset {}", documents.size(), offset); - - if (!documents.isEmpty() && documents.get(0) != null) { - BeneficiaryDocument firstDoc = documents.get(0); - logger.info("Sample doc - benId: {}, name: {} {}, district: {}, state: {}", - firstDoc.getBenId(), - firstDoc.getFirstName(), - firstDoc.getLastName(), - firstDoc.getDistrictName(), - firstDoc.getStateName()); - } int docsAddedInThisBatch = 0; for (BeneficiaryDocument doc : documents) { @@ -153,13 +156,13 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { successCount += indexed; failureCount += (esBatch.size() - indexed); processedCount += esBatch.size(); - + logger.info("✓ Indexed {}/{} documents successfully", indexed, esBatch.size()); esBatch.clear(); } } else { - logger.warn("Skipping document - doc null: {}, benId null: {}", - doc == null, doc != null ? (doc.getBenId() == null) : "N/A"); + logger.warn("Skipping document - doc null: {}, benId null: {}", + doc == null, doc != null ? (doc.getBenId() == null) : "N/A"); failureCount++; processedCount++; } @@ -169,9 +172,9 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { processedCount++; } } - + logger.info("Added {} documents to ES batch in this iteration", docsAddedInThisBatch); - + int notFetched = benRegIds.size() - documents.size(); if (notFetched > 0) { failureCount += notFetched; @@ -181,37 +184,58 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { offset += BATCH_SIZE; batchCounter++; - - logger.info("=== BATCH {} END: Processed={}, Success={}, Failed={} ===", - batchCounter, processedCount, successCount, failureCount); - - if (batchCounter % STATUS_UPDATE_FREQUENCY == 0) { - logger.info("Updating job progress (every {} batches)...", STATUS_UPDATE_FREQUENCY); - updateJobProgress(job, processedCount, successCount, failureCount, - offset, totalCount, startTime); + + logger.info("=== BATCH {} END: Processed={}, Success={}, Failed={} ===", + batchCounter, processedCount, successCount, failureCount); + + // Save progress every batch for resume capability + long now = System.currentTimeMillis(); + if (batchCounter % STATUS_UPDATE_FREQUENCY == 0 || (now - lastProgressUpdate) > 30000) { + logger.info("Saving checkpoint for resume capability..."); + updateJobProgress(job, processedCount, successCount, failureCount, + offset, totalCount, startTime); + lastProgressUpdate = now; } + // Brief pause every 10 batches if (batchCounter % 10 == 0) { logger.debug("Pausing for 500ms after {} batches", batchCounter); Thread.sleep(500); } } catch (Exception e) { - logger.error("!!! ERROR in batch at offset {}: {} !!!", offset, e.getMessage(), e); + consecutiveErrors++; + logger.error("!!! ERROR #{} in batch at offset {}: {} !!!", + consecutiveErrors, offset, e.getMessage(), e); logger.error("Exception type: {}", e.getClass().getName()); - logger.error("Stack trace:", e); - + + // Save progress before handling error job.setCurrentOffset(offset); job.setProcessedRecords(processedCount); job.setSuccessCount(successCount); job.setFailureCount(failureCount); syncJobRepository.save(job); - + + // If too many consecutive errors, mark job as STALLED for manual intervention + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + logger.error("TOO MANY CONSECUTIVE ERRORS ({}). Marking job as STALLED.", consecutiveErrors); + job.setStatus("STALLED"); + job.setErrorMessage("Too many consecutive errors at offset " + offset + ": " + e.getMessage()); + syncJobRepository.save(job); + return; + } + + // Skip this batch and continue offset += BATCH_SIZE; - Thread.sleep(2000); + + // Exponential backoff: wait longer after each error + long waitTime = Math.min(10000, 1000 * (long) Math.pow(2, consecutiveErrors)); + logger.info("Waiting {}ms before retry...", waitTime); + Thread.sleep(waitTime); } } + // Index remaining documents if (!esBatch.isEmpty()) { logger.info("Indexing final batch of {} documents", esBatch.size()); int indexed = bulkIndexDocuments(esBatch); @@ -220,21 +244,22 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { processedCount += esBatch.size(); } + // Mark as COMPLETED job.setStatus("COMPLETED"); job.setCompletedAt(new Timestamp(System.currentTimeMillis())); job.setProcessedRecords(processedCount); job.setSuccessCount(successCount); job.setFailureCount(failureCount); job.setCurrentOffset((int) totalCount); - + long duration = System.currentTimeMillis() - startTime; job.setProcessingSpeed(processedCount / (duration / 1000.0)); - + syncJobRepository.save(job); logger.info("Async sync job COMPLETED: jobId={}", jobId); - logger.info("Total: {}, Processed: {}, Success: {}, Failed: {}", - totalCount, processedCount, successCount, failureCount); + logger.info("Total: {}, Processed: {}, Success: {}, Failed: {}", + totalCount, processedCount, successCount, failureCount); logger.info("All 38+ beneficiary fields synced to Elasticsearch!"); } catch (Exception e) { @@ -247,6 +272,22 @@ public void syncAllBeneficiariesAsync(Long jobId, String triggeredBy) { } } + // Resume a stalled job + public void resumeStalledJob(Long jobId) { + logger.info("Attempting to resume stalled job: {}", jobId); + + ElasticsearchSyncJob job = syncJobRepository.findByJobId(jobId) + .orElseThrow(() -> new RuntimeException("Job not found: " + jobId)); + + if (!job.getStatus().equals("STALLED") && !job.getStatus().equals("RUNNING")) { + throw new RuntimeException("Cannot resume job with status: " + job.getStatus()); + } + + // Reset error counter and continue from last checkpoint + logger.info("Resuming from offset: {}", job.getCurrentOffset()); + syncAllBeneficiariesAsync(jobId, "AUTO_RESUME"); + } + /** * Helper method to safely convert various numeric types to BigInteger * CRITICAL: Native SQL queries return Long, not BigInteger @@ -256,7 +297,7 @@ private BigInteger toBigInteger(Object value) { logger.warn("Attempted to convert null value to BigInteger"); return null; } - + try { if (value instanceof BigInteger) { return (BigInteger) value; @@ -272,14 +313,14 @@ private BigInteger toBigInteger(Object value) { } return new BigInteger(value.toString()); } catch (NumberFormatException e) { - logger.error("Cannot convert '{}' (type: {}) to BigInteger: {}", - value, value.getClass().getName(), e.getMessage()); + logger.error("Cannot convert '{}' (type: {}) to BigInteger: {}", + value, value.getClass().getName(), e.getMessage()); return null; } } - private void updateJobProgress(ElasticsearchSyncJob job, long processed, long success, - long failure, int offset, long total, long startTime) { + private void updateJobProgress(ElasticsearchSyncJob job, long processed, long success, + long failure, int offset, long total, long startTime) { job.setProcessedRecords(processed); job.setSuccessCount(success); job.setFailureCount(failure); @@ -310,12 +351,10 @@ private int bulkIndexDocuments(List documents) { for (BeneficiaryDocument doc : documents) { if (doc.getBenId() != null) { br.operations(op -> op - .index(idx -> idx - .index(beneficiaryIndex) - .id(doc.getBenId()) - .document(doc) - ) - ); + .index(idx -> idx + .index(beneficiaryIndex) + .id(doc.getBenId()) + .document(doc))); } } @@ -327,8 +366,8 @@ private int bulkIndexDocuments(List documents) { if (item.error() == null) { successCount++; } else { - logger.error("ES indexing error for doc {}: {}", - item.id(), item.error().reason()); + logger.error("ES indexing error for doc {}: {}", + item.id(), item.error().reason()); } } } else { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index b65f435..aee0847 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -24,6 +24,8 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; + +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.SortOrder; import com.iemr.common.identity.repo.BenAddressRepo; @@ -56,11 +58,10 @@ public List> universalSearch(String query, Integer userId) { final Map userLocation = (userId != null) ? getUserLocation(userId) : null; boolean isNumeric = query.matches("\\d+"); - double minScore = isNumeric ? 1.0 : 2.0; + double minScore = isNumeric ? 1.0 : 1.5; SearchResponse response = esClient.search(s -> s .index(beneficiaryIndex) - .preference("_local") .requestCache(true) @@ -70,83 +71,120 @@ public List> universalSearch(String query, Integer userId) { .query(qq -> qq .bool(b -> { if (!isNumeric) { - // OPTIMIZED NAME SEARCH - // Use match_phrase_prefix for faster prefix matching - b.should(s1 -> s1.multiMatch(mm -> mm - .query(query) - .fields("firstName^3", "lastName^3") - .type(TextQueryType.Phrase) - .boost(3.0f))); - + // 1. EXACT MATCH (highest priority) + b.should(s1 -> s1.term(t -> t + .field("firstName.keyword") + .value(query) + .boost(20.0f))); b.should(s2 -> s2.term(t -> t + .field("lastName.keyword") + .value(query) + .boost(20.0f))); + + // 2. PREFIX MATCH (high priority for "sur" → "suraj") + b.should(s3 -> s3.prefix(p -> p .field("firstName.keyword") .value(query) .boost(10.0f))); - b.should(s3 -> s3.term(t -> t + b.should(s4 -> s4.prefix(p -> p .field("lastName.keyword") .value(query) .boost(10.0f))); - // Prefix search using index_prefixes (FAST!) - b.should(s4 -> s4.match(m -> m - .field("firstName.prefix") - .query(query) - .boost(2.0f))); + // 3. FUZZY MATCH (for typos: "vanit" → "vanitha") + // AUTO fuzziness: 1 edit for 3-5 chars, 2 edits for 6+ chars b.should(s5 -> s5.match(m -> m - .field("lastName.prefix") + .field("firstName") + .query(query) + .fuzziness("AUTO") + .prefixLength(1) // First char must match exactly + .maxExpansions(50) + .boost(5.0f))); + b.should(s6 -> s6.match(m -> m + .field("lastName") .query(query) - .boost(2.0f))); + .fuzziness("AUTO") + .prefixLength(1) + .maxExpansions(50) + .boost(5.0f))); + + // 4. WILDCARD MATCH (for "sur*" → "suraj", "surya") + if (query.length() >= 2) { + b.should(s7 -> s7.wildcard(w -> w + .field("firstName.keyword") + .value(query + "*") + .boost(8.0f))); + b.should(s8 -> s8.wildcard(w -> w + .field("lastName.keyword") + .value(query + "*") + .boost(8.0f))); + } + + // 5. CONTAINS MATCH (for partial matches anywhere) + if (query.length() >= 3) { + b.should(s9 -> s9.wildcard(w -> w + .field("firstName.keyword") + .value("*" + query + "*") + .boost(3.0f))); + b.should(s10 -> s10.wildcard(w -> w + .field("lastName.keyword") + .value("*" + query + "*") + .boost(3.0f))); + } } - b.should(s6 -> s6 - .term(t -> t.field("healthID").value(query).boost(15.0f))); - b.should(s7 -> s7 - .term(t -> t.field("abhaID").value(query).boost(15.0f))); - b.should(s8 -> s8 - .term(t -> t.field("beneficiaryID").value(query).boost(15.0f))); - b.should( - s9 -> s9.term(t -> t.field("benId").value(query).boost(15.0f))); - b.should(s10 -> s10 - .term(t -> t.field("aadharNo").value(query).boost(12.0f))); + // ID SEARCHES + b.should(s11 -> s11 + .term(t -> t.field("healthID").value(query).boost(25.0f))); + b.should(s12 -> s12 + .term(t -> t.field("abhaID").value(query).boost(25.0f))); + b.should(s13 -> s13 + .term(t -> t.field("beneficiaryID").value(query).boost(25.0f))); + b.should(s14 -> s14 + .term(t -> t.field("benId").value(query).boost(25.0f))); + b.should(s15 -> s15 + .term(t -> t.field("aadharNo").value(query).boost(20.0f))); if (isNumeric) { - // PREFIX QUERIES (much faster than wildcard) - b.should(s11 -> s11 - .prefix(p -> p.field("phoneNum").value(query).boost(5.0f))); - b.should(s12 -> s12 - .prefix(p -> p.field("healthID").value(query).boost(4.0f))); - b.should(s13 -> s13 - .prefix(p -> p.field("abhaID").value(query).boost(4.0f))); - b.should(s14 -> s14.prefix( - p -> p.field("beneficiaryID").value(query).boost(4.0f))); - - // ONLY use wildcard if query is long enough (>= 4 digits) + // PREFIX for phone/IDs + b.should(s16 -> s16 + .prefix(p -> p.field("phoneNum").value(query).boost(8.0f))); + b.should(s17 -> s17 + .prefix(p -> p.field("healthID").value(query).boost(6.0f))); + b.should(s18 -> s18 + .prefix(p -> p.field("abhaID").value(query).boost(6.0f))); + b.should(s19 -> s19.prefix( + p -> p.field("beneficiaryID").value(query).boost(6.0f))); + + // WILDCARD for phone contains if (query.length() >= 4) { - b.should(s15 -> s15.wildcard(w -> w + b.should(s20 -> s20.wildcard(w -> w .field("phoneNum") .value("*" + query + "*") - .boost(2.0f))); + .boost(3.0f))); } + // Numeric ID matches try { Long numericValue = Long.parseLong(query); - b.should(s16 -> s16.term(t -> t.field("benRegId") + b.should(s21 -> s21.term(t -> t.field("benRegId") + .value(numericValue).boost(25.0f))); + b.should(s22 -> s22.term(t -> t.field("benAccountID") .value(numericValue).boost(15.0f))); - b.should(s17 -> s17.term(t -> t.field("benAccountID") - .value(numericValue).boost(10.0f))); + // Location matching (if user location available) int intValue = numericValue.intValue(); if (userLocation != null) { Integer userVillageId = userLocation.get("villageId"); Integer userBlockId = userLocation.get("blockId"); if (userVillageId != null && userVillageId == intValue) { - b.should(s18 -> s18.term(t -> t.field("villageID") - .value(intValue).boost(3.0f))); + b.should(s23 -> s23.term(t -> t.field("villageID") + .value(intValue).boost(5.0f))); } if (userBlockId != null && userBlockId == intValue) { - b.should(s19 -> s19.term(t -> t.field("blockID") - .value(intValue).boost(2.0f))); + b.should(s24 -> s24.term(t -> t.field("blockID") + .value(intValue).boost(3.0f))); } } } catch (NumberFormatException e) { @@ -190,10 +228,11 @@ public List> universalSearch(String query, Integer userId) { if (!response.hits().hits().isEmpty()) { BeneficiariesESDTO firstResult = response.hits().hits().get(0).source(); - logger.info("First result - benRegId: {}, healthID: {}, abhaID: {}", + logger.info("Top result - score: {}, benRegId: {}, name: {} {}", + response.hits().hits().get(0).score(), firstResult.getBenRegId(), - firstResult.getHealthID(), - firstResult.getAbhaID()); + firstResult.getFirstName(), + firstResult.getLastName()); } if (response.hits().hits().isEmpty()) { diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index 0f80c12..498105a 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -28,8 +28,8 @@ public class ElasticsearchSyncService { private static final Logger logger = LoggerFactory.getLogger(ElasticsearchSyncService.class); // OPTIMIZED BATCH SIZES for maximum performance - private static final int DB_FETCH_SIZE = 5000; // Fetch 5000 IDs at once - private static final int ES_BULK_SIZE = 2000; // Index 2000 documents per bulk request + private static final int DB_FETCH_SIZE = 10000; + private static final int ES_BULK_SIZE = 5000; @Autowired private ElasticsearchClient esClient; @@ -244,7 +244,9 @@ private int bulkIndexDocuments(List documents) { } } - BulkResponse result = esClient.bulk(br.build()); + BulkResponse result = esClient.bulk( + br.refresh(Refresh.WaitFor).build() +); int successCount = 0; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5001e41..4a8a63c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -7,120 +7,151 @@ spring.jpa.hibernate.naming.implicit-strategy=org.hibernate.boot.model.naming.Im spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl # The SQL dialect makes Hibernate generate better SQL for the chosen database -spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.MySQLDialect +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQLDialect -spring.datasource.hikari.leak-detection-threshold=60000 +# ============================================================================ +# OPTIMIZED HIKARICP SETTINGS FOR HIGH CONCURRENCY +# ============================================================================ -# MySQL specific settings to prevent connection timeout -spring.datasource.hikari.data-source-properties.cachePrepStmts=true -spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250 -spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048 -spring.datasource.hikari.data-source-properties.useServerPrepStmts=true -spring.datasource.hikari.data-source-properties.autoReconnect=true -spring.datasource.hikari.data-source-properties.tcpKeepAlive=true -# spring.datasource.hikari.test-on-borrow=true - -# Reduce batch size for faster feedback -spring.jpa.properties.hibernate.jdbc.fetch_size=10000 -spring.jpa.properties.hibernate.jdbc.batch_size=5000 -spring.jpa.properties.hibernate.order_inserts=true -spring.jpa.properties.hibernate.order_updates=true +# Connection Pool Size - CRITICAL for handling multiple concurrent requests +# Rule: (max-pool-size) = (number of CPUs * 2) + effectiveSpindles +# For 8 CPUs: 8*2 + 4 = 20-30 is optimal +spring.datasource.hikari.maximum-pool-size=40 +spring.datasource.hikari.minimum-idle=20 -# Query timeout -spring.jpa.properties.hibernate.query.timeout=60000 +# Connection Timeouts - REDUCED for faster failure detection +spring.datasource.hikari.connection-timeout=20000 +spring.datasource.hikari.validation-timeout=3000 -spring.datasource.hikari.maximum-pool-size=30 -spring.datasource.hikari.minimum-idle=15 -spring.datasource.hikari.connection-timeout=30000 +# Idle/Lifetime - Prevents stale connections spring.datasource.hikari.idle-timeout=300000 -spring.datasource.hikari.max-lifetime=600000 -spring.datasource.hikari.keepalive-time=120000 -spring.datasource.hikari.validation-timeout=5000 -spring.datasource.hikari.leak-detection-threshold=60000 +spring.datasource.hikari.max-lifetime=1800000 +spring.datasource.hikari.keepalive-time=60000 +# Leak Detection - CRITICAL for debugging connection issues +spring.datasource.hikari.leak-detection-threshold=30000 -ring.datasource.hikari.data-source-properties.cachePrepStmts=true +# MySQL Connection Properties - Keeps connections alive +spring.datasource.hikari.data-source-properties.cachePrepStmts=true spring.datasource.hikari.data-source-properties.prepStmtCacheSize=500 spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=4096 spring.datasource.hikari.data-source-properties.useServerPrepStmts=true +spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true +spring.datasource.hikari.data-source-properties.maintainTimeStats=false + +# CRITICAL: Prevents MySQL "connection lost" errors spring.datasource.hikari.data-source-properties.autoReconnect=true spring.datasource.hikari.data-source-properties.tcpKeepAlive=true -spring.datasource.hikari.data-source-properties.useSSL=false +spring.datasource.hikari.data-source-properties.socketTimeout=60000 +spring.datasource.hikari.data-source-properties.connectTimeout=10000 + +# Health check +spring.datasource.hikari.health-check-properties.connectivityCheckTimeoutMs=3000 -spring.datasource.hikari.health-check-properties.mysql5Validation=true +# ============================================================================ +# HIBERNATE PERFORMANCE TUNING +# ============================================================================ + +# Batch processing for bulk operations +spring.jpa.properties.hibernate.jdbc.fetch_size=100 +spring.jpa.properties.hibernate.jdbc.batch_size=50 +spring.jpa.properties.hibernate.order_inserts=true +spring.jpa.properties.hibernate.order_updates=true -#Below lines are added for security reasons +# Query timeout (60 seconds) +spring.jpa.properties.hibernate.query.timeout=60000 + +# Statement cache +spring.jpa.properties.hibernate.query.plan_cache_max_size=2048 +spring.jpa.properties.hibernate.query.plan_parameter_metadata_max_size=128 + +# ============================================================================ +# REDIS SESSION MANAGEMENT +# ============================================================================ spring.session.store-type=redis spring.redis.password= spring.redis.port=6379 -## Below values are needed for extending the expiry time and extend expiry time. +# Redis connection pool +spring.redis.lettuce.pool.max-active=20 +spring.redis.lettuce.pool.max-idle=10 +spring.redis.lettuce.pool.min-idle=5 +spring.redis.lettuce.pool.max-wait=3000ms + +# Session expiry iemr.extend.expiry.time=true iemr.session.expiry.time=7200 enableIPValidation=false -#logging.pattern.level=DEBUG -logging.level.root=INFO -logging.level.org.springframework.web=INFO -logging.level.org.hibernate=INFO -logging.level.com.iemr=DEBUG -logging.level.org.springframework=INFO -logging.level.com.zaxxer.hikari=DEBUG - -door-to-door-page-size=2 +# ============================================================================ +# ELASTICSEARCH CONFIGURATION - OPTIMIZED +# ============================================================================ -#Get-HRP-Status -get-HRP-Status=ANC/getHRPStatus +# Connection timeouts - Balanced for reliability +elasticsearch.connection.timeout=5000 +elasticsearch.socket.timeout=30000 +elasticsearch.max.retry.timeout=60000 -#Get Beneficiary ABHA -getHealthID=healthID/getBenhealthID - -spring.main.allow-bean-definition-overriding=true -spring.main.allow-circular-references=true -jwt.access.expiration=86400000 -jwt.refresh.expiration=604800000 - -# Connection timeouts -elasticsearch.connection.timeout=10000 -elasticsearch.socket.timeout=120000 -elasticsearch.max.retry.timeout=120000 - -# Connection pooling - INCREASED for bulk operations -elasticsearch.max.connections=200 -elasticsearch.max.connections.per.route=100 +# Connection pooling - INCREASED for concurrent searches +elasticsearch.max.connections=100 +elasticsearch.max.connections.per.route=50 # Request configuration -elasticsearch.request.timeout=60000 +elasticsearch.request.timeout=10000 elasticsearch.max.result.window=10000 -# Bulk indexing - OPTIMIZED for maximum throughput -elasticsearch.bulk.size=2000 -elasticsearch.bulk.concurrent.requests=8 -elasticsearch.bulk.flush.interval=30s +# Bulk indexing - OPTIMIZED for background sync +elasticsearch.bulk.size=1000 +elasticsearch.bulk.concurrent.requests=4 +elasticsearch.bulk.flush.interval=10s -# Search performance (unchanged - maintains fast search) +# Search performance - FAST response for user queries elasticsearch.search.default.size=100 elasticsearch.search.max.size=500 elasticsearch.search.timeout=5s -# Query cache (for search performance) +# Query cache - CRITICAL for repeated searches elasticsearch.query.cache.enabled=true -elasticsearch.query.cache.size=10% +elasticsearch.query.cache.size=15% -# Request cache +# Request cache - Speeds up identical queries elasticsearch.request.cache.enabled=true -# Circuit breaker +# Circuit breaker - Prevents OOM elasticsearch.circuit.breaker.enabled=true -elasticsearch.circuit.breaker.limit=95% +elasticsearch.circuit.breaker.limit=90% # Async operations thread pool -elasticsearch.async.thread.pool.size=20 -elasticsearch.async.thread.pool.queue.size=5000 +elasticsearch.async.thread.pool.size=10 +elasticsearch.async.thread.pool.queue.size=2000 -# LOGGING (Reduce noise during sync) -logging.level.com.iemr.common.identity.service.elasticsearch=INFO -logging.level.org.elasticsearch=WARN -logging.level.org.hibernate.SQL=WARN +elasticsearch.bulk.refresh.interval=5 + +# ============================================================================ +# LOGGING - Balanced for debugging +# ============================================================================ +logging.level.root=INFO +logging.level.org.springframework.web=INFO +logging.level.org.hibernate=INFO +logging.level.com.iemr=DEBUG +logging.level.org.springframework=INFO logging.level.com.zaxxer.hikari=INFO +logging.level.com.zaxxer.hikari.pool.HikariPool=DEBUG + +# Log slow queries (helps identify bottlenecks) +logging.level.org.hibernate.SQL=WARN +spring.jpa.properties.hibernate.show_sql=false + +# ============================================================================ +# APPLICATION SPECIFIC +# ============================================================================ +door-to-door-page-size=2 +get-HRP-Status=ANC/getHRPStatus +getHealthID=healthID/getBenhealthID + +spring.main.allow-bean-definition-overriding=true +spring.main.allow-circular-references=true + +jwt.access.expiration=86400000 +jwt.refresh.expiration=604800000 From a68912d2dec24bfc96201ec097eee33b2bef7ad6 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 23 Jan 2026 19:38:33 +0530 Subject: [PATCH 13/14] fix: add middlename, maritalstatus --- .../common/identity/IdentityApplication.java | 2 + .../controller/IdentityESController.java | 6 +- .../elasticsearch/BeneficiaryDocument.java | 9 + .../identity/dto/BeneficiariesESDTO.java | 9 + .../common/identity/repo/BenDetailRepo.java | 114 ++++--- .../identity/service/IdentityService.java | 8 +- .../BeneficiaryDocumentDataService.java | 4 + .../ElasticsearchIndexingService.java | 310 +++++++++--------- .../elasticsearch/ElasticsearchService.java | 182 +++++----- 9 files changed, 357 insertions(+), 287 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/IdentityApplication.java b/src/main/java/com/iemr/common/identity/IdentityApplication.java index 2b6ac32..462c186 100644 --- a/src/main/java/com/iemr/common/identity/IdentityApplication.java +++ b/src/main/java/com/iemr/common/identity/IdentityApplication.java @@ -26,10 +26,12 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import com.iemr.common.identity.utils.IEMRApplBeans; @SpringBootApplication +@ComponentScan(basePackages = {"com.iemr.common.identity"}) public class IdentityApplication extends SpringBootServletInitializer { public static void main(String[] args) { diff --git a/src/main/java/com/iemr/common/identity/controller/IdentityESController.java b/src/main/java/com/iemr/common/identity/controller/IdentityESController.java index ca64beb..7a96f85 100644 --- a/src/main/java/com/iemr/common/identity/controller/IdentityESController.java +++ b/src/main/java/com/iemr/common/identity/controller/IdentityESController.java @@ -89,6 +89,8 @@ public ResponseEntity> advanceSearchBeneficiariesES( logger.info("Search params = {}", searchParams); String firstName = getString(searchParams, "firstName"); + String middleName = getString(searchParams, "middleName"); + String maritalStatus = getString(searchParams, "maritalStatus"); String lastName = getString(searchParams, "lastName"); Integer genderId = getInteger(searchParams, "genderId"); Date dob = getDate(searchParams, "dob"); @@ -115,9 +117,9 @@ public ResponseEntity> advanceSearchBeneficiariesES( Map searchResults = idService.advancedSearchBeneficiariesES( - firstName, lastName, genderId, dob, + firstName, middleName, lastName, genderId, dob, stateId, districtId, blockId, villageId, - fatherName, spouseName, phoneNumber, + fatherName, spouseName, maritalStatus, phoneNumber, beneficiaryId, healthId, aadharNo, userID, null, is1097 ); diff --git a/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java b/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java index aad9011..ec8df83 100644 --- a/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java +++ b/src/main/java/com/iemr/common/identity/data/elasticsearch/BeneficiaryDocument.java @@ -19,6 +19,9 @@ public class BeneficiaryDocument { @JsonProperty("firstName") private String firstName; + @JsonProperty("middleName") + private String middleName; + @JsonProperty("lastName") private String lastName; @@ -28,6 +31,12 @@ public class BeneficiaryDocument { @JsonProperty("spouseName") private String spouseName; + @JsonProperty("maritalStatusID") + private Integer maritalStatusID; + + @JsonProperty("maritalStatusName") + private String maritalStatusName; + @JsonProperty("age") private Integer age; diff --git a/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java b/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java index 6238e64..04cbaa4 100644 --- a/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java +++ b/src/main/java/com/iemr/common/identity/dto/BeneficiariesESDTO.java @@ -24,6 +24,9 @@ public class BeneficiariesESDTO { @JsonProperty("firstName") private String firstName; + + @JsonProperty("middleName") + private String middleName; @JsonProperty("lastName") private String lastName; @@ -56,6 +59,12 @@ public class BeneficiariesESDTO { @JsonProperty("spouseName") private String spouseName; + @JsonProperty("maritalStatusID") + private Integer maritalStatusID; + + @JsonProperty("maritalStatusName") + private String maritalStatusName; + @JsonProperty("createdBy") private String createdBy; diff --git a/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java b/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java index 085002b..ba68d0f 100644 --- a/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java +++ b/src/main/java/com/iemr/common/identity/repo/BenDetailRepo.java @@ -155,35 +155,39 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "m.BenRegId, " + // 0 "brm.beneficiaryID, " + // 1 "d.FirstName, " + // 2 - "d.LastName, " + // 3 - "d.GenderID, " + // 4 - "g.GenderName, " + // 5 - "d.DOB, " + // 6 - "TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) as Age, " + // 7 - "d.FatherName, " + // 8 - "d.SpouseName, " + // 9 - "d.IsHIVPositive, " + // 10 - "m.CreatedBy, " + // 11 - "m.CreatedDate, " + // 12 - "UNIX_TIMESTAMP(m.LastModDate) * 1000, " + // 13 - "m.BenAccountID, " + // 14 - "addr.CurrStateId, " + // 15 - "addr.CurrState, " + // 16 - "addr.CurrDistrictId, " + // 17 - "addr.CurrDistrict, " + // 18 - "addr.CurrSubDistrictId, " + // 19 - "addr.CurrSubDistrict, " + // 20 - "addr.CurrPinCode, " + // 21 - "addr.CurrServicePointId, " + // 22 - "addr.CurrServicePoint, " + // 23 - "addr.ParkingPlaceID, " + // 24 - "contact.PreferredPhoneNum " + // 25 - "addr.CurrVillageId, " + // 26 - "addr.CurrVillage " + // 27 + "d.MiddleName, " + // 3 + "d.LastName, " + // 4 + "d.GenderID, " + // 5 + "g.GenderName, " + // 6 + "d.DOB, " + // 7 + "TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) as Age, " + // 8 + "d.FatherName, " + // 9 + "d.SpouseName, " + // 10 + "d.MaritalStatusID, " + // 11 + "ms.Status as MaritalStatusName, " + // 12 + "d.IsHIVPositive, " + // 13 + "m.CreatedBy, " + // 14 + "m.CreatedDate, " + // 15 + "UNIX_TIMESTAMP(m.LastModDate) * 1000, " + // 16 + "m.BenAccountID, " + // 17 + "addr.CurrStateId, " + // 18 + "addr.CurrState, " + // 19 + "addr.CurrDistrictId, " + // 20 + "addr.CurrDistrict, " + // 21 + "addr.CurrSubDistrictId, " + // 22 + "addr.CurrSubDistrict, " + // 23 + "addr.CurrPinCode, " + // 24 + "addr.CurrServicePointId, " + // 25 + "addr.CurrServicePoint, " + // 26 + "addr.ParkingPlaceID, " + // 27 + "contact.PreferredPhoneNum " + // 28 + "addr.CurrVillageId, " + // 29 + "addr.CurrVillage " + // 30 "FROM i_beneficiarymapping m " + "LEFT JOIN i_beneficiarydetails d ON m.BenDetailsId = d.BeneficiaryDetailsID " + "LEFT JOIN m_beneficiaryregidmapping brm ON brm.BenRegId = m.BenRegId " + "LEFT JOIN db_iemr.m_gender g ON d.GenderID = g.GenderID " + + "LEFT JOIN db_iemr.m_maritalstatus ms ON d.MaritalStatusID = ms.StatusID " + "LEFT JOIN i_beneficiaryaddress addr ON m.BenAddressId = addr.BenAddressID " + "LEFT JOIN i_beneficiarycontacts contact ON m.BenContactsId = contact.BenContactsID " + "WHERE m.BenRegId IN (:ids) AND m.Deleted = false", @@ -197,6 +201,7 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "m.BenRegId, " + "brm.beneficiaryID, " + "d.FirstName, " + + "d.MiddleName, " + "d.LastName, " + "d.GenderID, " + "g.GenderName, " + @@ -204,6 +209,8 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) as Age, " + "d.FatherName, " + "d.SpouseName, " + + "d.MaritalStatusID, " + + "ms.Status as MaritalStatusName, " + "d.IsHIVPositive, " + "m.CreatedBy, " + "m.CreatedDate, " + @@ -226,9 +233,11 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "LEFT JOIN i_beneficiarydetails d ON m.BenDetailsId = d.BeneficiaryDetailsID " + "LEFT JOIN db_iemr.m_gender g ON d.GenderID = g.GenderID " + "LEFT JOIN m_beneficiaryregidmapping brm ON brm.BenRegId = m.BenRegId " + + "LEFT JOIN db_iemr.m_maritalstatus ms ON d.MaritalStatusID = ms.StatusID " + "LEFT JOIN i_beneficiaryaddress addr ON m.BenAddressId = addr.BenAddressID " + "LEFT JOIN i_beneficiarycontacts contact ON m.BenContactsId = contact.BenContactsID " + "WHERE (d.FirstName LIKE CONCAT('%', :query, '%') " + + " OR d.MiddleName LIKE CONCAT('%', :query, '%') " + " OR d.LastName LIKE CONCAT('%', :query, '%') " + " OR d.FatherName LIKE CONCAT('%', :query, '%') " + " OR d.BeneficiaryRegID = :query " + @@ -288,29 +297,32 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "m.BenRegId, " + // 0 "brm.beneficiaryID, " + // 1 "d.FirstName, " + // 2 - "d.LastName, " + // 3 - "d.GenderID, " + // 4 - "g.GenderName, " + // 5 - "d.DOB, " + // 6 - "TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) AS Age, "+// 7 - "d.FatherName, " + // 8 - "d.SpouseName, " + // 9 - "d.IsHIVPositive, " + // 10 - "m.CreatedBy, " + // 11 - "m.CreatedDate, " + // 12 - "UNIX_TIMESTAMP(m.LastModDate) * 1000, " + // 13 - "m.BenAccountID, " + // 14 - "addr.CurrStateId, " + // 15 - "addr.CurrState, " + // 16 - "addr.CurrDistrictId, " + // 17 - "addr.CurrDistrict, " + // 18 - "addr.CurrSubDistrictId, " + // 19 - "addr.CurrSubDistrict, " + // 20 - "addr.CurrPinCode, " + // 21 - "addr.CurrServicePointId, " + // 22 - "addr.CurrServicePoint, " + // 23 - "addr.ParkingPlaceID, " + // 24 - "contact.PreferredPhoneNum " + // 25 + "d.MiddleName, " + // 3 + "d.LastName, " + // 4 + "d.GenderID, " + // 5 + "g.GenderName, " + // 6 + "d.DOB, " + // 7 + "TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) AS Age, "+// 8 + "d.FatherName, " + // 9 + "d.SpouseName, " + // 10 + "d.MaritalStatusID, " + // 11 + "ms.Status AS MaritalStatusName, " + // 12 + "d.IsHIVPositive, " + // 13 + "m.CreatedBy, " + // 14 + "m.CreatedDate, " + // 15 + "UNIX_TIMESTAMP(m.LastModDate) * 1000, " + // 16 + "m.BenAccountID, " + // 17 + "addr.CurrStateId, " + // 18 + "addr.CurrState, " + // 19 + "addr.CurrDistrictId, " + // 20 + "addr.CurrDistrict, " + // 21 + "addr.CurrSubDistrictId, " + // 22 + "addr.CurrSubDistrict, " + // 23 + "addr.CurrPinCode, " + // 24 + "addr.CurrServicePointId, " + // 25 + "addr.CurrServicePoint, " + // 26 + "addr.ParkingPlaceID, " + // 27 + "contact.PreferredPhoneNum " + // 28 "addr.CurrVillageId, " + "addr.CurrVillage " + "FROM i_beneficiarymapping m " + @@ -318,6 +330,7 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi " ON m.BenDetailsId = d.BeneficiaryDetailsID " + "LEFT JOIN db_iemr.m_gender g " + " ON d.GenderID = g.GenderID " + + "LEFT JOIN db_iemr.m_maritalstatus ms ON d.MaritalStatusID = ms.StatusID " + "LEFT JOIN i_beneficiaryaddress addr " + " ON m.BenAddressId = addr.BenAddressID " + "LEFT JOIN i_beneficiarycontacts contact " + @@ -326,6 +339,7 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "WHERE m.Deleted = false " + "AND (:firstName IS NULL OR d.FirstName LIKE CONCAT('%', :firstName, '%')) " + + "AND (:middleName IS NULL OR d.MiddleName LIKE CONCAT('%', :middleName, '%')) " + "AND (:lastName IS NULL OR d.LastName LIKE CONCAT('%', :lastName, '%')) " + "AND (:genderId IS NULL OR d.GenderID = :genderId) " + "AND (:dob IS NULL OR DATE(d.DOB) = DATE(:dob)) " + @@ -336,7 +350,7 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi "AND (:fatherName IS NULL OR d.FatherName LIKE CONCAT('%', :fatherName, '%')) " + "AND (:spouseName IS NULL OR d.SpouseName LIKE CONCAT('%', :spouseName, '%')) " + - + "AND (:maritalStatus IS NULL OR d.MaritalStatusID = :maritalStatus) " + "AND (:phoneNumber IS NULL OR " + " contact.PreferredPhoneNum LIKE CONCAT('%', :phoneNumber, '%') OR " + " contact.PhoneNum1 LIKE CONCAT('%', :phoneNumber, '%') OR " + @@ -352,6 +366,7 @@ int untagFamily(@Param("modifiedBy") String modifiedBy, @Param("vanSerialNo") Bi nativeQuery = true) List advancedSearchBeneficiaries( @Param("firstName") String firstName, + @Param("middleName") String middleName, @Param("lastName") String lastName, @Param("genderId") Integer genderId, @Param("dob") Date dob, @@ -360,6 +375,7 @@ List advancedSearchBeneficiaries( @Param("blockId") Integer blockId, @Param("fatherName") String fatherName, @Param("spouseName") String spouseName, + @Param("maritalStatus") String maritalStatus, @Param("phoneNumber") String phoneNumber, @Param("beneficiaryId") String beneficiaryId ); diff --git a/src/main/java/com/iemr/common/identity/service/IdentityService.java b/src/main/java/com/iemr/common/identity/service/IdentityService.java index 6db2101..08b9fbb 100644 --- a/src/main/java/com/iemr/common/identity/service/IdentityService.java +++ b/src/main/java/com/iemr/common/identity/service/IdentityService.java @@ -585,9 +585,9 @@ public List getBeneficiariesByPhoneNum(String phoneNum) * Advanced search using Elasticsearch with fallback to database */ public Map advancedSearchBeneficiariesES( - String firstName, String lastName, Integer genderId, java.util.Date dob, + String firstName, String middleName, String lastName, Integer genderId, java.util.Date dob, Integer stateId, Integer districtId, Integer blockId, Integer villageId, - String fatherName, String spouseName, String phoneNumber, + String fatherName, String spouseName, String maritalStatus, String phoneNumber, String beneficiaryId, String healthId, String aadharNo, Integer userId, String auth, Boolean is1097) throws Exception { @@ -602,8 +602,8 @@ public Map advancedSearchBeneficiariesES( // Call Elasticsearch service List> esResults = elasticsearchService.advancedSearch( - firstName, lastName, genderId, dob, stateId, districtId, - blockId, villageId, fatherName, spouseName, phoneNumber, + firstName, middleName, lastName, genderId, dob, stateId, districtId, + blockId, villageId, fatherName, spouseName, maritalStatus, phoneNumber, beneficiaryId, healthId, aadharNo, userId ); diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java index cdca328..d11ac0f 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/BeneficiaryDocumentDataService.java @@ -229,6 +229,7 @@ private BeneficiaryDocument mapRowToDocument(Object[] row) { doc.setBeneficiaryID(beneficiaryID); doc.setFirstName(getString(row[idx++])); + doc.setMiddleName(getString(row[idx++])); doc.setLastName(getString(row[idx++])); doc.setGenderID(getInteger(row[idx++])); doc.setGenderName(getString(row[idx++])); @@ -237,6 +238,9 @@ private BeneficiaryDocument mapRowToDocument(Object[] row) { doc.setAge(getInteger(row[idx++])); doc.setFatherName(getString(row[idx++])); doc.setSpouseName(getString(row[idx++])); + + doc.setMaritalStatusID(getInteger(row[idx++])); + doc.setMaritalStatusName(getString(row[idx++])); doc.setIsHIVPos(getString(row[idx++])); doc.setCreatedBy(getString(row[idx++])); diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java index 44d13d3..eea47b8 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchIndexingService.java @@ -16,237 +16,231 @@ @Service public class ElasticsearchIndexingService { - + private static final Logger logger = LoggerFactory.getLogger(ElasticsearchIndexingService.class); - + @Autowired private ElasticsearchClient esClient; - + @Autowired private ElasticsearchSyncService syncService; - + @Value("${elasticsearch.index.beneficiary}") private String beneficiaryIndex; - + /** * Create index optimized for BULK INDEXING * Settings will be updated for search after sync completes */ public void createIndexWithMapping() throws Exception { logger.info("Creating index optimized for bulk indexing: {}", beneficiaryIndex); - + // Delete existing index if it exists if (esClient.indices().exists(e -> e.index(beneficiaryIndex)).value()) { logger.warn("Index {} already exists, deleting...", beneficiaryIndex); esClient.indices().delete(d -> d.index(beneficiaryIndex)); } - + // PHASE 1 SETTINGS: Optimized for BULK INDEXING (maximum write speed) IndexSettings settings = IndexSettings.of(s -> s - // CRITICAL: Disable refresh during bulk indexing - .refreshInterval(t -> t.time("-1")) // -1 = disable completely for max speed - - // Use 1 shard for datasets < 50GB (optimal for 784K records) - .numberOfShards("1") - - // No replicas during initial indexing (add later for HA) - .numberOfReplicas("0") - - // Disable query cache during indexing - .queries(q -> q - .cache(c -> c.enabled(false)) - ) - - .maxResultWindow(10000) - - // CRITICAL: Async translog for maximum speed - .translog(t -> t - .durability(TranslogDurability.Async) - .syncInterval(ts -> ts.time("120s")) // Longer interval for bulk ops - ) - ); - + // CRITICAL: Disable refresh during bulk indexing + .refreshInterval(t -> t.time("-1")) // -1 = disable completely for max speed + + // Use 1 shard for datasets < 50GB (optimal for 784K records) + .numberOfShards("1") + + // No replicas during initial indexing (add later for HA) + .numberOfReplicas("0") + + // Disable query cache during indexing + .queries(q -> q + .cache(c -> c.enabled(false))) + + .maxResultWindow(10000) + + // CRITICAL: Async translog for maximum speed + .translog(t -> t + .durability(TranslogDurability.Async) + .syncInterval(ts -> ts.time("120s")) // Longer interval for bulk ops + )); + // Field mappings (supports fast search) TypeMapping mapping = TypeMapping.of(tm -> tm - .properties("benId", Property.of(p -> p.keyword(k -> k))) - .properties("benRegId", Property.of(p -> p.long_(l -> l))) - .properties("beneficiaryID", Property.of(p -> p.keyword(k -> k))) - - .properties("firstName", Property.of(p -> p.text(t -> t - .analyzer("standard") - .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) - .fields("prefix", Property.of(fp -> fp.text(txt -> txt - .analyzer("standard") - .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) - ))) - ))) - - .properties("lastName", Property.of(p -> p.text(t -> t - .analyzer("standard") - .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) - .fields("prefix", Property.of(fp -> fp.text(txt -> txt - .analyzer("standard") - .indexPrefixes(ip -> ip.minChars(2).maxChars(5)) - ))) - ))) - - .properties("fatherName", Property.of(p -> p.text(t -> t - .analyzer("standard") - .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) - ))) - - .properties("spouseName", Property.of(p -> p.text(t -> t - .analyzer("standard") - .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) - ))) - - .properties("genderID", Property.of(p -> p.integer(i -> i))) - .properties("genderName", Property.of(p -> p.keyword(k -> k))) - .properties("dOB", Property.of(p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis")))) - .properties("age", Property.of(p -> p.integer(i -> i))) - - .properties("phoneNum", Property.of(p -> p.keyword(k -> k - .fields("ngram", Property.of(fp -> fp.text(txt -> txt - .analyzer("standard") - .searchAnalyzer("standard") - ))) - ))) - - .properties("isHIVPos", Property.of(p -> p.keyword(k -> k))) - .properties("createdBy", Property.of(p -> p.keyword(k -> k))) - .properties("createdDate", Property.of(p -> p.date(d -> d))) - .properties("lastModDate", Property.of(p -> p.long_(l -> l))) - .properties("benAccountID", Property.of(p -> p.long_(l -> l))) - - .properties("healthID", Property.of(p -> p.keyword(k -> k))) - .properties("abhaID", Property.of(p -> p.keyword(k -> k))) - .properties("abhaCreatedDate", Property.of(p -> p.keyword(k -> k))) - .properties("familyID", Property.of(p -> p.keyword(k -> k))) - - // Geographic fields - .properties("stateID", Property.of(p -> p.integer(i -> i))) - .properties("stateName", Property.of(p -> p.keyword(k -> k))) - .properties("districtID", Property.of(p -> p.integer(i -> i))) - .properties("districtName", Property.of(p -> p.keyword(k -> k))) - .properties("blockID", Property.of(p -> p.integer(i -> i))) - .properties("blockName", Property.of(p -> p.keyword(k -> k))) - .properties("villageID", Property.of(p -> p.integer(i -> i))) - .properties("villageName", Property.of(p -> p.keyword(k -> k))) - .properties("pinCode", Property.of(p -> p.keyword(k -> k))) - .properties("servicePointID", Property.of(p -> p.integer(i -> i))) - .properties("servicePointName", Property.of(p -> p.keyword(k -> k))) - .properties("parkingPlaceID", Property.of(p -> p.integer(i -> i))) - - // Permanent address fields - .properties("permStateID", Property.of(p -> p.integer(i -> i))) - .properties("permStateName", Property.of(p -> p.keyword(k -> k))) - .properties("permDistrictID", Property.of(p -> p.integer(i -> i))) - .properties("permDistrictName", Property.of(p -> p.keyword(k -> k))) - .properties("permBlockID", Property.of(p -> p.integer(i -> i))) - .properties("permBlockName", Property.of(p -> p.keyword(k -> k))) - .properties("permVillageID", Property.of(p -> p.integer(i -> i))) - .properties("permVillageName", Property.of(p -> p.keyword(k -> k))) - - // Identity fields - .properties("aadharNo", Property.of(p -> p.keyword(k -> k))) - .properties("govtIdentityNo", Property.of(p -> p.keyword(k -> k))) - ); - + .properties("benId", Property.of(p -> p.keyword(k -> k))) + .properties("benRegId", Property.of(p -> p.long_(l -> l))) + .properties("beneficiaryID", Property.of(p -> p.keyword(k -> k))) + + .properties("firstName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + .fields("prefix", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)))))))) + + .properties("middleName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + .fields("prefix", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)))))))) + + .properties("lastName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256)))) + .fields("prefix", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .indexPrefixes(ip -> ip.minChars(2).maxChars(5)))))))) + + .properties("fatherName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))))) + + .properties("spouseName", Property.of(p -> p.text(t -> t + .analyzer("standard") + .fields("keyword", Property.of(fp -> fp.keyword(k -> k.ignoreAbove(256))))))) + + .properties("maritalStatusID", Property.of(p -> p.integer(i -> i))) + .properties("maritalStatusName", Property.of(p -> p.keyword(k -> k))) + + .properties("genderID", Property.of(p -> p.integer(i -> i))) + .properties("genderName", Property.of(p -> p.keyword(k -> k))) + .properties("dOB", Property.of(p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis")))) + .properties("age", Property.of(p -> p.integer(i -> i))) + + .properties("phoneNum", Property.of(p -> p.keyword(k -> k + .fields("ngram", Property.of(fp -> fp.text(txt -> txt + .analyzer("standard") + .searchAnalyzer("standard"))))))) + + .properties("isHIVPos", Property.of(p -> p.keyword(k -> k))) + .properties("createdBy", Property.of(p -> p.keyword(k -> k))) + .properties("createdDate", Property.of(p -> p.date(d -> d))) + .properties("lastModDate", Property.of(p -> p.long_(l -> l))) + .properties("benAccountID", Property.of(p -> p.long_(l -> l))) + + .properties("healthID", Property.of(p -> p.keyword(k -> k))) + .properties("abhaID", Property.of(p -> p.keyword(k -> k))) + .properties("abhaCreatedDate", Property.of(p -> p.keyword(k -> k))) + .properties("familyID", Property.of(p -> p.keyword(k -> k))) + + // Geographic fields + .properties("stateID", Property.of(p -> p.integer(i -> i))) + .properties("stateName", Property.of(p -> p.keyword(k -> k))) + .properties("districtID", Property.of(p -> p.integer(i -> i))) + .properties("districtName", Property.of(p -> p.keyword(k -> k))) + .properties("blockID", Property.of(p -> p.integer(i -> i))) + .properties("blockName", Property.of(p -> p.keyword(k -> k))) + .properties("villageID", Property.of(p -> p.integer(i -> i))) + .properties("villageName", Property.of(p -> p.keyword(k -> k))) + .properties("pinCode", Property.of(p -> p.keyword(k -> k))) + .properties("servicePointID", Property.of(p -> p.integer(i -> i))) + .properties("servicePointName", Property.of(p -> p.keyword(k -> k))) + .properties("parkingPlaceID", Property.of(p -> p.integer(i -> i))) + + // Permanent address fields + .properties("permStateID", Property.of(p -> p.integer(i -> i))) + .properties("permStateName", Property.of(p -> p.keyword(k -> k))) + .properties("permDistrictID", Property.of(p -> p.integer(i -> i))) + .properties("permDistrictName", Property.of(p -> p.keyword(k -> k))) + .properties("permBlockID", Property.of(p -> p.integer(i -> i))) + .properties("permBlockName", Property.of(p -> p.keyword(k -> k))) + .properties("permVillageID", Property.of(p -> p.integer(i -> i))) + .properties("permVillageName", Property.of(p -> p.keyword(k -> k))) + + // Identity fields + .properties("aadharNo", Property.of(p -> p.keyword(k -> k))) + .properties("govtIdentityNo", Property.of(p -> p.keyword(k -> k)))); + esClient.indices().create(c -> c - .index(beneficiaryIndex) - .settings(settings) - .mappings(mapping) - ); - + .index(beneficiaryIndex) + .settings(settings) + .mappings(mapping)); + logger.info("Index created with BULK INDEXING optimization"); logger.info("Settings: refresh=disabled, replicas=0, async_translog, 1 shard"); } - + /** * PHASE 2: Optimize for SEARCH after bulk indexing completes * Call this AFTER indexAllBeneficiaries() finishes */ public void optimizeForSearch() throws Exception { logger.info("PHASE 2: Optimizing index for SEARCH performance"); - + // Step 1: Force refresh to make all documents searchable logger.info("Step 1/3: Forcing refresh to make documents visible..."); esClient.indices().refresh(r -> r.index(beneficiaryIndex)); logger.info("Documents are now searchable"); - + // Step 2: Update settings for production search logger.info("Step 2/3: Updating index settings for production..."); esClient.indices().putSettings(s -> s - .index(beneficiaryIndex) - .settings(is -> is - .refreshInterval(t -> t.time("1s")) // Enable 1s refresh for near real-time search - .numberOfReplicas("1") // Add replica for high availability - .translog(t -> t - .durability(TranslogDurability.Request) // Synchronous for data safety - .syncInterval(ts -> ts.time("5s")) - ) - .queries(q -> q - .cache(c -> c.enabled(true)) // Enable query cache for faster searches - ) - ) - ); + .index(beneficiaryIndex) + .settings(is -> is + .refreshInterval(t -> t.time("1s")) // Enable 1s refresh for near real-time search + .numberOfReplicas("1") // Add replica for high availability + .translog(t -> t + .durability(TranslogDurability.Request) // Synchronous for data safety + .syncInterval(ts -> ts.time("5s"))) + .queries(q -> q + .cache(c -> c.enabled(true)) // Enable query cache for faster searches + ))); logger.info("Settings applied: refresh=1s, replicas=1, query_cache=enabled"); - + // Step 3: Force merge to optimize segments logger.info("Step 3/3: Force merging segments for optimal read performance..."); logger.info("This may take 5-15 minutes depending on data size..."); esClient.indices().forcemerge(f -> f - .index(beneficiaryIndex) - .maxNumSegments(1L) // Single segment per shard = fastest searches - .flush(true) - ); + .index(beneficiaryIndex) + .maxNumSegments(1L) // Single segment per shard = fastest searches + .flush(true)); logger.info("Segments merged to 1 per shard"); - + logger.info("INDEX OPTIMIZATION COMPLETE!"); logger.info("Index is now ready for searches"); } - + /** * COMPLETE WORKFLOW: Create index + Sync data + Optimize * This is your existing endpoint, now with automatic optimization */ public Map indexAllBeneficiaries() { logger.info("COMPLETE INDEXING WORKFLOW"); - + long startTime = System.currentTimeMillis(); - + try { // Execute bulk indexing (now uses optimized batch queries) logger.info("PHASE 1: Bulk indexing beneficiaries with batch queries..."); ElasticsearchSyncService.SyncResult result = syncService.syncAllBeneficiaries(); - + long indexingTime = System.currentTimeMillis() - startTime; - logger.info("Bulk indexing completed in {} seconds ({} minutes)", - indexingTime / 1000, indexingTime / 60000); + logger.info("Bulk indexing completed in {} seconds ({} minutes)", + indexingTime / 1000, indexingTime / 60000); logger.info("Success: {}, Failed: {}", result.getSuccessCount(), result.getFailureCount()); - + // Optimize for search logger.info(""); logger.info("PHASE 2: Optimizing for search..."); long optimizeStart = System.currentTimeMillis(); optimizeForSearch(); long optimizeTime = System.currentTimeMillis() - optimizeStart; - logger.info("Optimization completed in {} seconds ({} minutes)", - optimizeTime / 1000, optimizeTime / 60000); - + logger.info("Optimization completed in {} seconds ({} minutes)", + optimizeTime / 1000, optimizeTime / 60000); + long totalTime = System.currentTimeMillis() - startTime; - + logger.info("COMPLETE WORKFLOW FINISHED!"); logger.info("Total time: {} seconds ({} minutes)", totalTime / 1000, totalTime / 60000); logger.info("Indexing: {}m | Optimization: {}m", indexingTime / 60000, optimizeTime / 60000); - + // Return response in your existing format Map response = new HashMap<>(); response.put("success", result.getSuccessCount()); response.put("failed", result.getFailureCount()); - + return response; - + } catch (Exception e) { logger.error("Error during indexing workflow", e); Map response = new HashMap<>(); @@ -255,21 +249,21 @@ public Map indexAllBeneficiaries() { return response; } } - + /** * Get index statistics (unchanged) */ public Map getIndexStats() throws Exception { var stats = esClient.indices().stats(s -> s.index(beneficiaryIndex)); var settings = esClient.indices().getSettings(g -> g.index(beneficiaryIndex)); - + Map info = new HashMap<>(); info.put("documentCount", stats.indices().get(beneficiaryIndex).primaries().docs().count()); info.put("sizeInBytes", stats.indices().get(beneficiaryIndex).primaries().store().sizeInBytes()); info.put("refreshInterval", settings.get(beneficiaryIndex).settings().index().refreshInterval().time()); info.put("numberOfShards", settings.get(beneficiaryIndex).settings().index().numberOfShards()); info.put("numberOfReplicas", settings.get(beneficiaryIndex).settings().index().numberOfReplicas()); - + return info; } } \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java index aee0847..31afd1a 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchService.java @@ -77,30 +77,45 @@ public List> universalSearch(String query, Integer userId) { .value(query) .boost(20.0f))); b.should(s2 -> s2.term(t -> t + .field("middleName.keyword") + .value(query) + .boost(20.0f))); + b.should(s3 -> s3.term(t -> t .field("lastName.keyword") .value(query) .boost(20.0f))); // 2. PREFIX MATCH (high priority for "sur" → "suraj") - b.should(s3 -> s3.prefix(p -> p + b.should(s4 -> s4.prefix(p -> p .field("firstName.keyword") .value(query) .boost(10.0f))); - b.should(s4 -> s4.prefix(p -> p + b.should(s5 -> s5.prefix(p -> p + .field("middleName.keyword") + .value(query) + .boost(10.0f))); + b.should(s6 -> s6.prefix(p -> p .field("lastName.keyword") .value(query) .boost(10.0f))); // 3. FUZZY MATCH (for typos: "vanit" → "vanitha") // AUTO fuzziness: 1 edit for 3-5 chars, 2 edits for 6+ chars - b.should(s5 -> s5.match(m -> m + b.should(s7 -> s7.match(m -> m .field("firstName") .query(query) .fuzziness("AUTO") .prefixLength(1) // First char must match exactly .maxExpansions(50) .boost(5.0f))); - b.should(s6 -> s6.match(m -> m + b.should(s8 -> s8.match(m -> m + .field("middleName") + .query(query) + .fuzziness("AUTO") + .prefixLength(1) // First char must match exactly + .maxExpansions(50) + .boost(5.0f))); + b.should(s9 -> s9.match(m -> m .field("lastName") .query(query) .fuzziness("AUTO") @@ -110,11 +125,15 @@ public List> universalSearch(String query, Integer userId) { // 4. WILDCARD MATCH (for "sur*" → "suraj", "surya") if (query.length() >= 2) { - b.should(s7 -> s7.wildcard(w -> w + b.should(s10 -> s10.wildcard(w -> w .field("firstName.keyword") .value(query + "*") .boost(8.0f))); - b.should(s8 -> s8.wildcard(w -> w + b.should(s11 -> s11.wildcard(w -> w + .field("middleName.keyword") + .value(query + "*") + .boost(8.0f))); + b.should(s12 -> s12.wildcard(w -> w .field("lastName.keyword") .value(query + "*") .boost(8.0f))); @@ -122,11 +141,15 @@ public List> universalSearch(String query, Integer userId) { // 5. CONTAINS MATCH (for partial matches anywhere) if (query.length() >= 3) { - b.should(s9 -> s9.wildcard(w -> w + b.should(s13 -> s13.wildcard(w -> w .field("firstName.keyword") .value("*" + query + "*") .boost(3.0f))); - b.should(s10 -> s10.wildcard(w -> w + b.should(s14 -> s14.wildcard(w -> w + .field("middleName.keyword") + .value("*" + query + "*") + .boost(3.0f))); + b.should(s15 -> s15.wildcard(w -> w .field("lastName.keyword") .value("*" + query + "*") .boost(3.0f))); @@ -134,31 +157,31 @@ public List> universalSearch(String query, Integer userId) { } // ID SEARCHES - b.should(s11 -> s11 + b.should(s16 -> s16 .term(t -> t.field("healthID").value(query).boost(25.0f))); - b.should(s12 -> s12 + b.should(s17 -> s17 .term(t -> t.field("abhaID").value(query).boost(25.0f))); - b.should(s13 -> s13 + b.should(s18 -> s18 .term(t -> t.field("beneficiaryID").value(query).boost(25.0f))); - b.should(s14 -> s14 + b.should(s19 -> s19 .term(t -> t.field("benId").value(query).boost(25.0f))); - b.should(s15 -> s15 + b.should(s20 -> s20 .term(t -> t.field("aadharNo").value(query).boost(20.0f))); if (isNumeric) { // PREFIX for phone/IDs - b.should(s16 -> s16 + b.should(s21 -> s21 .prefix(p -> p.field("phoneNum").value(query).boost(8.0f))); - b.should(s17 -> s17 + b.should(s22 -> s22 .prefix(p -> p.field("healthID").value(query).boost(6.0f))); - b.should(s18 -> s18 + b.should(s23 -> s23 .prefix(p -> p.field("abhaID").value(query).boost(6.0f))); - b.should(s19 -> s19.prefix( + b.should(s24 -> s24.prefix( p -> p.field("beneficiaryID").value(query).boost(6.0f))); // WILDCARD for phone contains if (query.length() >= 4) { - b.should(s20 -> s20.wildcard(w -> w + b.should(s25 -> s25.wildcard(w -> w .field("phoneNum") .value("*" + query + "*") .boost(3.0f))); @@ -167,9 +190,9 @@ public List> universalSearch(String query, Integer userId) { // Numeric ID matches try { Long numericValue = Long.parseLong(query); - b.should(s21 -> s21.term(t -> t.field("benRegId") + b.should(s26 -> s26.term(t -> t.field("benRegId") .value(numericValue).boost(25.0f))); - b.should(s22 -> s22.term(t -> t.field("benAccountID") + b.should(s27 -> s27.term(t -> t.field("benAccountID") .value(numericValue).boost(15.0f))); // Location matching (if user location available) @@ -179,11 +202,11 @@ public List> universalSearch(String query, Integer userId) { Integer userBlockId = userLocation.get("blockId"); if (userVillageId != null && userVillageId == intValue) { - b.should(s23 -> s23.term(t -> t.field("villageID") + b.should(s28 -> s28.term(t -> t.field("villageID") .value(intValue).boost(5.0f))); } if (userBlockId != null && userBlockId == intValue) { - b.should(s24 -> s24.term(t -> t.field("blockID") + b.should(s29 -> s29.term(t -> t.field("blockID") .value(intValue).boost(3.0f))); } } @@ -206,13 +229,13 @@ public List> universalSearch(String query, Integer userId) { .source(src -> src .filter(f -> f - .includes("benRegId", "beneficiaryID", "firstName", "lastName", + .includes("benRegId", "beneficiaryID", "firstName", "middleName", "lastName", "genderID", "genderName", "dOB", "phoneNum", "stateID", "stateName", "districtID", "blockID", "villageID", "healthID", "abhaID", "abhaCreatedDate", "familyID", - "fatherName", "spouseName", "age", "createdBy", "createdDate", + "fatherName", "spouseName","maritalStatusID", "maritalStatusName", "age", "createdBy", "createdDate", "lastModDate", "benAccountID", "districtName", "blockName", "villageName", "pinCode", "servicePointID", "servicePointName", "parkingPlaceID", "permStateID", "permStateName", "permDistrictID", @@ -295,6 +318,7 @@ private List getFunctionScores(Map userLocation) */ public List> advancedSearch( String firstName, + String middleName, String lastName, Integer genderId, Date dob, @@ -304,6 +328,7 @@ public List> advancedSearch( Integer villageId, String fatherName, String spouseName, + String maritalStatus, String phoneNumber, String beneficiaryId, String healthId, @@ -346,6 +371,14 @@ public List> advancedSearch( .match(mm -> mm.field("firstName").query(firstName).boost(2.0f))) .minimumShouldMatch("1"))); } + if (middleName != null && !middleName.trim().isEmpty()) { + b.must(m -> m.bool(bb -> bb + .should(s1 -> s1.term( + t -> t.field("middleName.keyword").value(middleName).boost(5.0f))) + .should(s2 -> s2 + .match(mm -> mm.field("middleName").query(middleName).boost(2.0f))) + .minimumShouldMatch("1"))); + } if (lastName != null && !lastName.trim().isEmpty()) { b.must(m -> m.bool(bb -> bb @@ -386,8 +419,8 @@ public List> advancedSearch( , BeneficiariesESDTO.class); if (response.hits().hits().isEmpty()) { - return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, - stateId, districtId, blockId, villageId, fatherName, spouseName, + return searchInDatabaseForAdvanced(firstName, middleName, lastName, genderId, dob, + stateId, districtId, blockId, villageId, fatherName, spouseName, maritalStatus, phoneNumber, beneficiaryId, healthId, aadharNo); } @@ -398,8 +431,8 @@ public List> advancedSearch( } catch (Exception e) { logger.error("ES advanced search failed: {}", e.getMessage()); - return searchInDatabaseForAdvanced(firstName, lastName, genderId, dob, - stateId, districtId, blockId, villageId, fatherName, spouseName, + return searchInDatabaseForAdvanced(firstName, middleName, lastName, genderId, dob, + stateId, districtId, blockId, villageId, fatherName, spouseName, maritalStatus, phoneNumber, beneficiaryId, healthId, aadharNo); } } @@ -408,15 +441,15 @@ public List> advancedSearch( * Database fallback for advanced search */ private List> searchInDatabaseForAdvanced( - String firstName, String lastName, Integer genderId, Date dob, + String firstName, String middleName, String lastName, Integer genderId, Date dob, Integer stateId, Integer districtId, Integer blockId, Integer villageId, - String fatherName, String spouseName, String phoneNumber, - String beneficiaryId, String healthId, String aadharNo) { + String fatherName, String spouseName, String maritalStatus, + String phoneNumber, String beneficiaryId, String healthId, String aadharNo) { try { List results = benDetailRepo.advancedSearchBeneficiaries( - firstName, lastName, genderId, dob, stateId, districtId, - blockId, fatherName, spouseName, phoneNumber, + firstName, middleName, lastName, genderId, dob, stateId, districtId, + blockId, fatherName, spouseName, maritalStatus, phoneNumber, beneficiaryId); return results.stream() @@ -555,17 +588,20 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat result.put("beneficiaryRegID", esData.getBenRegId()); result.put("beneficiaryID", esData.getBeneficiaryID()); result.put("firstName", esData.getFirstName()); + result.put("middleName", esData.getMiddleName() != null ? esData.getMiddleName() : ""); result.put("lastName", esData.getLastName() != null ? esData.getLastName() : ""); result.put("genderID", esData.getGenderID()); result.put("genderName", esData.getGenderName()); result.put("dob", esData.getDOB()); result.put("dOB", esData.getDOB()); result.put("age", esData.getAge()); - result.put("actualAge", esData.getAge()); // NEW - result.put("ageUnits", "Years"); // NEW + result.put("actualAge", esData.getAge()); + result.put("ageUnits", "Years"); result.put("fatherName", esData.getFatherName() != null ? esData.getFatherName() : ""); result.put("spouseName", esData.getSpouseName() != null ? esData.getSpouseName() : ""); + result.put("maritalStatusID", esData.getMaritalStatusID() != null ? esData.getMaritalStatusID() : ""); + result.put("maritalStatusName", esData.getMaritalStatusName() != null ? esData.getMaritalStatusName() : ""); result.put("isHIVPos", esData.getIsHIVPos() != null ? esData.getIsHIVPos() : ""); result.put("createdBy", esData.getCreatedBy()); @@ -575,11 +611,7 @@ private Map mapESResultToExpectedFormat(BeneficiariesESDTO esDat // Family ID - use both formats result.put("familyID", esData.getFamilyID()); - result.put("familyId", esData.getFamilyID()); // NEW - alternate key - - // Marital status - NEW - result.put("maritalStatusName", null); // TODO: Add to ES if available - result.put("maritalStatus", new HashMap<>()); + result.put("familyId", esData.getFamilyID()); // Head of family - NEW (add these fields to BeneficiariesESDTO if available) result.put("headOfFamily_RelationID", null); @@ -754,57 +786,59 @@ private Map mapToExpectedFormat(Object[] row) { Long beneficiaryRegID = getLong(row[0]); String beneficiaryID = getString(row[1]); String firstName = getString(row[2]); - String lastName = getString(row[3]); - Integer genderID = getInteger(row[4]); - String genderName = getString(row[5]); - Date dob = getDate(row[6]); - Integer age = getInteger(row[7]); - String fatherName = getString(row[8]); - String spouseName = getString(row[9]); - String isHIVPos = getString(row[10]); - String createdBy = getString(row[11]); - Date createdDate = getDate(row[12]); - Long lastModDate = getLong(row[13]); - Long benAccountID = getLong(row[14]); - - Integer stateID = getInteger(row[15]); - String stateName = getString(row[16]); - Integer districtID = getInteger(row[17]); - String districtName = getString(row[18]); - Integer blockID = getInteger(row[19]); - String blockName = getString(row[20]); - String pinCode = getString(row[21]); - Integer servicePointID = getInteger(row[22]); - String servicePointName = getString(row[23]); - Integer parkingPlaceID = getInteger(row[24]); - String phoneNum = getString(row[25]); - - Integer villageID = getInteger(row[26]); - String villageName = getString(row[27]); + String middleName = getString(row[3]); + String lastName = getString(row[4]); + Integer genderID = getInteger(row[5]); + String genderName = getString(row[6]); + Date dob = getDate(row[7]); + Integer age = getInteger(row[8]); + String fatherName = getString(row[9]); + String spouseName = getString(row[10]); + String maritalStatusID = getString(row[11]); + String maritalStatusName = getString(row[12]); + String isHIVPos = getString(row[13]); + String createdBy = getString(row[14]); + Date createdDate = getDate(row[15]); + Long lastModDate = getLong(row[16]); + Long benAccountID = getLong(row[17]); + + Integer stateID = getInteger(row[18]); + String stateName = getString(row[19]); + Integer districtID = getInteger(row[20]); + String districtName = getString(row[21]); + Integer blockID = getInteger(row[22]); + String blockName = getString(row[23]); + String pinCode = getString(row[24]); + Integer servicePointID = getInteger(row[25]); + String servicePointName = getString(row[26]); + Integer parkingPlaceID = getInteger(row[27]); + String phoneNum = getString(row[28]); + + Integer villageID = getInteger(row[29]); + String villageName = getString(row[30]); result.put("beneficiaryRegID", beneficiaryRegID); result.put("beneficiaryID", beneficiaryID); result.put("firstName", firstName); + result.put("middleName", middleName != null ? middleName : ""); result.put("lastName", lastName != null ? lastName : ""); result.put("genderID", genderID); result.put("genderName", genderName); result.put("dOB", dob); result.put("dob", dob); result.put("age", age); - result.put("actualAge", age); // NEW - result.put("ageUnits", "Years"); // NEW + result.put("actualAge", age); + result.put("ageUnits", "Years"); result.put("fatherName", fatherName != null ? fatherName : ""); result.put("spouseName", spouseName != null ? spouseName : ""); + result.put("maritalStatusID", maritalStatusID != null ? maritalStatusID : ""); + result.put("maritalStatusName", maritalStatusName != null ? maritalStatusName : ""); result.put("isHIVPos", isHIVPos != null ? isHIVPos : ""); result.put("createdBy", createdBy); result.put("createdDate", createdDate); result.put("lastModDate", lastModDate); result.put("benAccountID", benAccountID); - - // NEW fields - result.put("maritalStatusName", null); - result.put("maritalStatus", new HashMap<>()); - + Map mGender = new HashMap<>(); mGender.put("genderID", genderID); mGender.put("genderName", genderName); From 332b70f14cd361c15b06142ee8a94f625b34d138 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 27 Jan 2026 09:45:46 +0530 Subject: [PATCH 14/14] fix: remove refresh while bulk indexing --- .../service/elasticsearch/ElasticsearchSyncService.java | 9 ++++++--- src/main/resources/application.properties | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java index 498105a..cd1e234 100644 --- a/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java +++ b/src/main/java/com/iemr/common/identity/service/elasticsearch/ElasticsearchSyncService.java @@ -244,9 +244,12 @@ private int bulkIndexDocuments(List documents) { } } - BulkResponse result = esClient.bulk( - br.refresh(Refresh.WaitFor).build() -); +// BulkResponse result = esClient.bulk( +// br.refresh(Refresh.WaitFor).build() +// ); + + BulkResponse result = esClient.bulk(br.build()); + int successCount = 0; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4a8a63c..12be1b4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -102,7 +102,7 @@ elasticsearch.max.result.window=10000 # Bulk indexing - OPTIMIZED for background sync elasticsearch.bulk.size=1000 -elasticsearch.bulk.concurrent.requests=4 +elasticsearch.bulk.concurrent.requests=6 elasticsearch.bulk.flush.interval=10s # Search performance - FAST response for user queries