HibernateSearch Error while trying to fetch data

Hibernate Search
1

Hi,

My spring-boot application to fetch data from elasticsearch db is failing with the following error.
Hibernate ORM mapping:
type ‘a.b.c.x.LearningCurve’:
attribute ‘dynamic’:
failures:
- Invalid value. Expected ‘STRICT’, actual is ‘null’
field ‘_entity_type’:
failures:
- Missing property mapping

Following is the mapping that is present for the indices
{
“learnings” : {
“mappings” : {
“properties” : {
“ack” : {
“type” : “boolean”
},
“ackTimestamp” : {
“type” : “long”
},
“ackedBy” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“additionalInfo” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“cancelledBy” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“cancelledTimestamp” : {
“type” : “long”
},
“count” : {
“type” : “long”
},
“memberId” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“memberName” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“memberType” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“memberClass” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnGroup” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnId” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnKey” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnName” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnSource” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“learnTime” : {
“type” : “long”
},
“firstOccurrence” : {
“type” : “long”
},
“learnEntries” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“lastOccurrence” : {
“type” : “long”
},
“owner” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“primaryAddress” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“medium” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“sequenceId” : {
“type” : “long”
},
“severity” : {
“type” : “integer”
},
“spec” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“summary” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“timeToCancel” : {
“type” : “long”
},
“type” : {
“type” : “integer”
},
“courseList” : {
“type” : “text”,
“fields” : {
“keyword” : {
“type” : “keyword”,
“ignore_above” : 256
}
}
},
“version” : {
“type” : “integer”
}
}
}
}
}

And following is my Entity Class.

import java.io.Serializable;
import java.util.Objects;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;

import org.elasticsearch.common.Nullable;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.stereotype.Component;

@Entity
//@Document(indexName=“fm.active_learns”)
@Indexed (index=“fm.active_learns”)
@Table(name = “active_learns”, schema = “fm”)
@Component
public class DBActiveLearns implements Serializable {

/**
 *
 */
private static final long serialVersionUID = 1L;

@Field
@Column(name = "learnId")
@Nullable
private String learnId;

@Field
@Column(name = "learnName")
private String learnName;

// int value 0/1/2
@Field
@Column(name = "type")
private Integer type;

@Field
@Column(name = "protocol")
private String protocol;

@Field
@Column(name = "version")
@Nullable
private Integer version;

@Field
@Column(name = "memberId")
private String memberId;

@Field
@Column(name = "memberType")
@Nullable
private String memberType;

@Field
@Column(name = "learnKey")
@Nullable
private String learnKey;

@Field
@Column(name = "learnGroup")
@Nullable
private String learnGroup;

@Field
@Column(name = "learnTime")
private Long learnTime;


@Field
@Column(name = "summary")
private String summary;

@Field
@Column(name = "severity")
private Integer severity;

@Field
@Column(name = "learnClass")
@Nullable
private String learnClass;


@Field
@Column(name = "ack")
private boolean ack;

@Field
@Column(name = "lastOccurrence")
private Long lastOccurrence;

@Field
@Column(name = "firstOccurrence")
private Long firstOccurrence;

@Field
@Column(name = "count")
@Nullable
private long count;


@Field
@Column(name = "ackTimestamp")
@Nullable
private Long ackTimestamp;


@Field
@Column(name = "cancelledTimestamp")
@Nullable
private Long cancelledTimestamp;

@Field
@Column(name = "ackedBy")
@Nullable
private String ackedBy;

@Field
@Column(name = "learnSource")
@Nullable
private String learnSource;

@Field
@Column(name = "courseList")
@Nullable
private String courseList;

@Field
@Column(name = "spec")
@Nullable
private String spec;


@Field
@Column(name = "additionalInfo")
@Nullable
private String additionalInfo;

@Id
@DocumentId
@Field
@Column(name = "sequenceId")
private long sequenceId;

@Field
@Column(name = "timeToCancel")
@Nullable
private long timeToCancel;


@Field
@Column(name = "memberName")
@Nullable
private String memberName;

@Field
@Column(name = "primaryAddress")
@Nullable
private String primaryAddress;

@Field
@Column(name = "cancelledBy")
private String cancelledBy = "NA";

@Field
@Column(name = "owner")
private String owner = "All";

@Field
@Column(name = "learnEntries")
@Nullable
private String learnEntries;

public DbBasedCorrelatedLearn() {
}

public DbBasedCorrelatedLearn(String learnId, String learnName, Integer type, String protocol, Integer version, String memberId, String memberType, String learnKey, String learnGroup, Long learnTime, String summary, Integer severity, String learnClass, boolean ack, Long lastOccurrence, Long firstOccurrence, long count, Long ackTimestamp, Long cancelledTimestamp, String ackedBy, String learnSourceIp, String varbindList, String spec, String additionalInfo, long sequenceId, long timeToCancel, String memberName, String primaryIp, String cancelledBy, String owner, String learnEntries) {
    this.learnId = learnId;
    this.learnName = learnName;
    this.type = type;
    this.protocol = protocol;
    this.version = version;
    this.memberId = memberId;
    this.memberType = memberType;
    this.learnKey = learnKey;
    this.learnGroup = learnGroup;
    this.learnTime = learnTime;
    this.summary = summary;
    this.severity = severity;
    this.learnClass = learnClass;
    this.ack = ack;
    this.lastOccurrence = lastOccurrence;
    this.firstOccurrence = firstOccurrence;
    this.count = count;
    this.ackTimestamp = ackTimestamp;
    this.cancelledTimestamp = cancelledTimestamp;
    this.ackedBy = ackedBy;
    this.learnSourceIp = learnSourceIp;
    this.varbindList = varbindList;
    this.spec = spec;
    this.additionalInfo = additionalInfo;
    this.sequenceId = sequenceId;
    this.timeToCancel = timeToCancel;
    this.memberName = memberName;
    this.primaryIp = primaryIp;
    this.cancelledBy = cancelledBy;
    this.owner = owner;
    this.learnEntries = learnEntries;
}

public String getOwner() {
    return owner;
}

public void setOwner(String owner) {
    this.owner = owner;
}

public String getLearnEntries() {
    return learnEntries;
}

public void setLearnEntries(String learnEntries) {
    this.learnEntries = learnEntries;
}

public String getLearnId() {
    return learnId;
}

public void setLearnId(String learnId) {
    this.learnId = learnId;
}

public String getLearnName() {
    return learnName;
}

public void setLearnName(String learnName) {
    this.learnName = learnName;
}

public Integer getType() {
    return type;
}

public void setType(Integer type) {
    this.type = type;
}

public String getProtocol() {
    return protocol;
}

public void setProtocol(String protocol) {
    this.protocol = protocol;
}

public Integer getVersion() {
    return version;
}

public void setVersion(Integer version) {
    this.version = version;
}

public String getMemberType() {
    return memberType;
}

public void setMemberType(String memberType) {
    this.memberType = memberType;
}

public String getLearnKey() {
    return learnKey;
}

public void setLearnKey(String learnKey) {
    this.learnKey = learnKey;
}

public String getLearnGroup() {
    return learnGroup;
}

public void setLearnGroup(String learnGroup) {
    this.learnGroup = learnGroup;
}

public Long getLearnTime() {
    return learnTime;
}

public void setLearnTime(Long learnTime) {
    this.learnTime = learnTime;
}

public String getSummary() {
    return summary;
}

public void setSummary(String summary) {
    this.summary = summary;
}

public String getLearnClass() {
    return learnClass;
}

public void setLearnClass(String learnClass) {
    this.learnClass = learnClass;
}

public boolean isAck() {
    return ack;
}

public void setAck(boolean ack) {
    this.ack = ack;
}

public Long getLastOccurrence() {
    return lastOccurrence;
}

public void setLastOccurrence(Long lastOccurrence) {
    this.lastOccurrence = lastOccurrence;
}

public Long getFirstOccurrence() {
    return firstOccurrence;
}

public void setFirstOccurrence(Long firstOccurrence) {
    this.firstOccurrence = firstOccurrence;
}

public long getCount() {
    return count;
}

public void setCount(long count) {
    this.count = count;
}

public Long getAckTimestamp() {
    return ackTimestamp;
}

public void setAckTimestamp(Long ackTimestamp) {
    this.ackTimestamp = ackTimestamp;
}

public String getAckedBy() {
    return ackedBy;
}

public void setAckedBy(String ackedBy) {
    this.ackedBy = ackedBy;
}

public String getLearnSourceIp() {
    return learnSourceIp;
}

public void setLearnSourceIp(String learnSourceIP) {
    this.learnSourceIp = learnSourceIP;
}

public String getCourseList() {
    return varbindList;
}

public void setCourseList(String varbindList) {
    this.varbindList = varbindList;
}

public String getSpec() {
    return spec;
}

public void setSpec(String spec) {
    this.spec = spec;
}

public String getAdditionalInfo() {
    return additionalInfo;
}

public void setAdditionalInfo(String additionalInfo) {
    this.additionalInfo = additionalInfo;
}

public long getSequenceId() {
    return sequenceId;
}

public void setSequenceId(long sequenceId) {
    this.sequenceId = sequenceId;
}

public long getTimeToCancel() {
    return timeToCancel;
}

public void setTimeToCancel(long timeToCancel) {
    this.timeToCancel = timeToCancel;
}

public String getMemberName() {
    return memberName;
}

public void setMemberName(String memberName) {
    this.memberName = memberName;
}

public String getPrimaryAddress() {
    return primaryIp;
}

public void setPrimaryAddress(String primaryIp) {
    this.primaryIp = primaryIp;
}

public String getCancelledBy() {
    return cancelledBy;
}

public void setCancelledBy(String cancelledBy) {
    this.cancelledBy = cancelledBy;
}

public Integer getSeverity() {
    return severity;
}

public void setSeverity(Integer severity) {
    this.severity = severity;
}

public Long getCancelledTimestamp() {
    return cancelledTimestamp;
}

public void setCancelledTimestamp(Long cancelledTimestamp) {
    this.cancelledTimestamp = cancelledTimestamp;
}

@Override
public String toString() {
    return "DbBasedCorrelatedLearn{" +
            "learnId='" + learnId + '\'' +
            ", learnName='" + learnName + '\'' +
            ", type=" + type +
            ", protocol='" + protocol + '\'' +
            ", version=" + version +
            ", memberId='" + memberId + '\'' +
            ", memberType='" + memberType + '\'' +
            ", learnKey='" + learnKey + '\'' +
            ", learnGroup='" + learnGroup + '\'' +
            ", learnTime=" + learnTime +
            ", summary='" + summary + '\'' +
            ", severity=" + severity +
            ", learnClass='" + learnClass + '\'' +
            ", ack=" + ack +
            ", lastOccurrence=" + lastOccurrence +
            ", firstOccurrence=" + firstOccurrence +
            ", count=" + count +
            ", ackTimestamp=" + ackTimestamp +
            ", cancelledTimestamp=" + cancelledTimestamp +
            ", ackedBy='" + ackedBy + '\'' +
            ", learnSourceIp='" + learnSourceIp + '\'' +
            ", varbindList='" + varbindList + '\'' +
            ", spec='" + spec + '\'' +
            ", additionalInfo='" + additionalInfo + '\'' +
            ", sequenceId=" + sequenceId +
            ", timeToCancel=" + timeToCancel +
            ", memberName='" + memberName + '\'' +
            ", primaryIp='" + primaryIp + '\'' +
            ", cancelledBy='" + cancelledBy + '\'' +
            ", owner='" + owner + '\'' +
            ", learnEntries='" + learnEntries + '\'' +
            '}';
}

@Override
public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;
    DbBasedCorrelatedLearn that = (DbBasedCorrelatedLearn) o;
    return ack == that.ack &&
            count == that.count &&
            sequenceId == that.sequenceId &&
            timeToCancel == that.timeToCancel &&
            Objects.equals(learnId, that.learnId) &&
            Objects.equals(learnName, that.learnName) &&
            Objects.equals(type, that.type) &&
            Objects.equals(protocol, that.protocol) &&
            Objects.equals(version, that.version) &&
            Objects.equals(memberId, that.memberId) &&
            Objects.equals(memberType, that.memberType) &&
            Objects.equals(learnKey, that.learnKey) &&
            Objects.equals(learnGroup, that.learnGroup) &&
            Objects.equals(learnTime, that.learnTime) &&
            Objects.equals(summary, that.summary) &&
            Objects.equals(severity, that.severity) &&
            Objects.equals(learnClass, that.learnClass) &&
            Objects.equals(lastOccurrence, that.lastOccurrence) &&
            Objects.equals(firstOccurrence, that.firstOccurrence) &&
            Objects.equals(ackTimestamp, that.ackTimestamp) &&
            Objects.equals(cancelledTimestamp, that.cancelledTimestamp) &&
            Objects.equals(ackedBy, that.ackedBy) &&
            Objects.equals(learnSourceIp, that.learnSourceIp) &&
            Objects.equals(varbindList, that.varbindList) &&
            Objects.equals(spec, that.spec) &&
            Objects.equals(additionalInfo, that.additionalInfo) &&
            Objects.equals(memberName, that.memberName) &&
            Objects.equals(primaryIp, that.primaryIp) &&
            Objects.equals(cancelledBy, that.cancelledBy) &&
            Objects.equals(owner, that.owner) &&
            Objects.equals(learnEntries, that.learnEntries);
}

@Override
public int hashCode() {
    return Objects.hash(learnId, learnName, type, protocol, version, memberId, memberType, learnKey, learnGroup, learnTime, summary, severity, learnClass, ack, lastOccurrence, firstOccurrence, count, ackTimestamp, cancelledTimestamp, ackedBy, learnSourceIp, varbindList, spec, additionalInfo, sequenceId, timeToCancel, memberName, primaryIp, cancelledBy, owner, learnEntries);
}

public String getMemberId() {
    return memberId;
}

public void setMemberId(String memberId) {
    this.memberId = memberId;
}

}

I some how feel there is a discripency in the way the entity class as is. The entity class should be used for both SQL and NoSQL DBs. It would be of great help if anyone tell me what is going wrong.

2

Hello,

You must be using an old version of Hibernate Search 6, because in more recent versions you would get this message at the top:

Validation of the existing index in the Elasticsearch cluster failed. See below for details

The problem you’re facing is that the schema of existing indexes in the Elasticsearch cluster does not match what Hibernate Search expects. In particular, the top-level attribute ‘dynamic’ is not set, while Hibernate Search expects it to be set to ‘strict’, and the field ‘_entity_type’ does not exist.

This probably happens because you’re updating from an older Beta version to a newer one.

The easiest solution would be to drop your indexes completely and recreate them. You can do that manually, or through Hibernate Search with schema management (to drop/recreate the indexes) and then mass indexing (to populate the indexes).

3

Hi,
Resolved the STRICT error by adding the mapping with dynamic property in index’s mapping.
When the application runs, the following error is seen because of the entity class

Caused by: org.hibernate.search.util.common.SearchException: HSEARCH000520: Hibernate Search encountered failures during bootstrap. Failures:

Hibernate ORM mapping: 
    type 'com.rbbn.ems.fm.model.correlator.DbActiveLearns': 
        field '_entity_type': 
            failures: 
              - Missing property mapping
at org.hibernate.search.engine.reporting.spi.RootFailureCollector.checkNoFailure(RootFailureCollector.java:50) ~[hibernate-search-engine-6.0.0.Beta8.jar:6.0.0.Beta8]
at org.hibernate.search.engine.common.impl.SearchIntegrationPartialBuildStateImpl$SearchIntegrationFinalizerImpl.finalizeIntegration(SearchIntegrationPartialBuildStateImpl.java:172) ~[hibernate-search-engine-6.0.0.Beta8.jar:6.0.0.Beta8]
at org.hibernate.search.mapper.orm.bootstrap.impl.HibernateOrmIntegrationBooterImpl.doBootSecondPhase(HibernateOrmIntegrationBooterImpl.java:276) ~[hibernate-search-mapper-orm-6.0.0.Beta8.jar:6.0.0.Beta8]
at org.hibernate.search.mapper.orm.bootstrap.impl.HibernateOrmIntegrationBooterImpl.bootNow(HibernateOrmIntegrationBooterImpl.java:198) ~[hibernate-search-mapper-orm-6.0.0.Beta8.jar:6.0.0.Beta8]
at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:642) ~[na:na]
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[na:na]
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[na:na]
at org.hibernate.search.mapper.orm.bootstrap.impl.HibernateSearchSessionFactoryObserver.sessionFactoryCreated(HibernateSearchSessionFactoryObserver.java:41) ~[hibernate-search-mapper-orm-6.0.0.Beta8.jar:6.0.0.Beta8]
at org.hibernate.internal.SessionFactoryObserverChain.sessionFactoryCreated(SessionFactoryObserverChain.java:35) ~[hibernate-core-5.4.17.Final.jar:5.4.17.Final]
at org.hibernate.internal.SessionFactoryImpl.<init>(SessionFactoryImpl.java:382) ~[hibernate-core-5.4.17.Final.jar:5.4.17.Final]
at org.hibernate.boot.internal.SessionFactoryBuilderImpl.build(SessionFactoryBuilderImpl.java:469) ~[hibernate-core-5.4.17.Final.jar:5.4.17.Final]
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.build(EntityManagerFactoryBuilderImpl.java:1259) ~[hibernate-core-5.4.17.Final.jar:5.4.17.Final]
at org.springframework.orm.jpa.vendor.SpringHibernateJpaPersistenceProvider.createContainerEntityManagerFactory(SpringHibernateJpaPersistenceProvider.java:58) ~[spring-orm-5.2.7.RELEASE.jar:5.2.7.RELEASE]
at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.createNativeEntityManagerFactory(LocalContainerEntityManagerFactoryBean.java:365) ~[spring-orm-5.2.7.RELEASE.jar:5.2.7.RELEASE]
at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.buildNativeEntityManagerFactory(AbstractEntityManagerFactoryBean.java:391) ~[spring-orm-5.2.7.RELEASE.jar:5.2.7.RELEASE]
... 21 common frames omitted

Tried different options in @Field annotation to get the best fit for the mappings of the indcices from ElasticSearch but the error is seen always. Can anyone tell me what is wrong with my entity class file.

The error specifying _entity_type is not at all present in the entity class or the mappings that the index have in Elasticsearch.

Thanks,
Srikanth C

4

I have overcome the error by introducing the property
spring.jpa.properties.hibernate.search.schema_management.strategy=none

But due to some discrepancy between the entity class and the schema, I could not get the documents from the index.

List learnEvents = new ArrayList();

	SearchSession searchSession = Search.session(entityManager);
	
	SearchResult<DbActiveLearns> activeEventResultSet = searchSession.search(DbActiveLearns.class)
			.where(f -> f.matchAll()).fetchAll();
	activeEvents = activeEventResultSet.hits();

The debug log says the following
2020-11-24 09:30:17.335 TRACE 6216 — [port thread - 2] o.h.search.elasticsearch.request : HSEARCH400093: Executed Elasticsearch HTTP POST request to path ‘/learnings-read/_search’ with query parameters {size=10000, track_total_hits=true} and 1 objects in payload in 27ms. Response had status 200 ‘OK’. Request body: <
{
“query”: {
“match_all”: POST
},
“_source”: false
}

. Response body: <
{
“took”: 20,
“timed_out”: false,
“_shards”: {
“total”: 1,
“successful”: 1,
“skipped”: 0,
“failed”: 0
},
“hits”: {
“total”: {
“value”: 1,
“relation”: “eq”
},
“max_score”: 1.0,
“hits”: [
{
“_index”: “learnings”,
“_type”: “_doc”,
“_id”: “XxZB0nUBmfRN3DNwXCtc”,
“_score”: 1.0
}
]
}
}
The actual document data in ElasticSearch is different.
I want to understand what is the difference between the entity class of mine with the mappings from Elasticsearch DB.

5

That’s what schema validation is trying to tell you.

In your Elasticsearch mapping, you’re missing a property named _entity_type for the index of the DbActiveLearns entity. The problem is not in your class, it’s in your Elasticsearch mapping.
_entity_type is a property that Hibernate Search adds automatically in order to know the type of every document.

If you are struggling to get a correct mapping, I would recommend letting Hibernate Search create it, as I explained in my previous post:

If you want to know what the mapping is supposed to look like, but do not want it to be created automatically, then just launch your application against a test Elasticsearch instance, with the hibernate.search.schema_management.strategy set to create. It will create schemas for you and you will be able to have a look at what it expects exactly.