Delta import is working but the result is not what we want.
Solr replaces all documents that were added with the full import and updates/adds the one that have been updated instead of only update/delete the necessary documents.
Is there some config option to keep the documents that don't have updates? Or can somebody send me a example data-config with working delta import so i can learn from it and see what i did wrong?
My data-config file to make things easier.
<dataConfig>
<dataSource convertType="true" type="JdbcDataSource"
driver="com.mysql.jdbc.Driver"
url="jdbc:mysql://localhost/table1"
user="user_db"
password="password"/>
<document name="content">
<entity name="id" pk="id"
query="SELECT pp.*, pp.profile_positions_id as id, p.position as position_name, b1.name as branche_name, b2.name as sub_branche_name, pc.name as position_class_name,
p1.user_id as profile_user_id, p1.company_id as company_id, p1.active as profile_active, p1.visibility as profile_visibility, p1.profile_type as profile_type, p1.gender as profile_gender, p1.initials as profile_initials, p1.firstname as profile_firstname, p1.infix as profile_infix, p1.lastname as profile_lastname, p1.profile_description as profile_description, p1.email as profile_email, p1.zip as profile_zip, p1.lat as profile_lat, p1.lon as profile_lon, p1.city as profile_city, p1.travel_ability as profile_travel_ability, p1.status as profile_status, p1.deleted as profile_deleted, p1.deleted as profile_active,
c.company_name as company_name, c.businesstype as company_businesstype, c.amount_of_people as company_amount_of_people, c.visibility as company_visibility, c.deleted as company_deleted, c.valid as company_valid, pp.level as level,
u.deleted as user_deleted, pp.deleted as position_deleted
FROM profile_positions pp
LEFT JOIN profiles p1 ON p1.profile_id = pp.profile_id
LEFT JOIN companies c ON c.company_id = p1.company_id
LEFT JOIN positions p ON p.position_id = pp.position_id
LEFT JOIN branches b1 ON b1.branche_id = pp.branche_id
LEFT JOIN branches b2 ON b2.branche_id = pp.sub_branche_id
LEFT JOIN position_classes pc ON pc.position_class_id = pp.position_class
LEFT JOIN users u ON u.user_id = p1.user_id;"
deltaImportQuery="SELECT pp.*, pp.profile_positions_id as id, p.position as position_name, b1.name as branche_name, b2.name as sub_branche_name, pc.name as position_class_name,
p1.user_id as profile_user_id, p1.company_id as company_id, p1.active as profile_active, p1.visibility as profile_visibility, p1.profile_type as profile_type, p1.gender as profile_gender, p1.initials as profile_initials, p1.firstname as profile_firstname, p1.infix as profile_infix, p1.lastname as profile_lastname, p1.profile_description as profile_description, p1.email as profile_email, p1.zip as profile_zip, p1.lat as profile_lat, p1.lon as profile_lon, p1.city as profile_city, p1.travel_ability as profile_travel_ability, p1.status as profile_status, p1.deleted as profile_deleted, p1.deleted as profile_active,
c.company_name as company_name, c.businesstype as company_businesstype, c.amount_of_people as company_amount_of_people, c.visibility as company_visibility, c.deleted as company_deleted, c.valid as company_valid, pp.level as level,
u.deleted as user_deleted, pp.deleted as position_deleted
FROM profile_positions pp
LEFT JOIN profiles p1 ON p1.profile_id = pp.profile_id
LEFT JOIN companies c ON c.company_id = p1.company_id
LEFT JOIN positions p ON p.position_id = pp.position_id
LEFT JOIN branches b1 ON b1.branche_id = pp.branche_id
LEFT JOIN branches b2 ON b2.branche_id = pp.sub_branche_id
LEFT JOIN position_classes pc ON pc.position_class_id = pp.position_class
LEFT JOIN users u ON u.user_id = p1.user_id
WHERE pp.profile_positions_id='${dih.delta.id}';"
deltaQuery="select profile_positions_id from profile_positions pp
LEFT JOIN profiles p1 ON p1.profile_id = pp.profile_id
where updated_at >= '${dih.last_index_time}' OR created_at >= '${dih.last_index_time}' OR company_id IN(
SELECT company_id FROM companies where updated_at >= '${dih.last_index_time}' OR created_at >= '${dih.last_index_time}') OR user_id IN(
SELECT user_id FROM users where updated_at >= '${dih.last_index_time}' OR created_at >= '${dih.last_index_time}');">
<field column="id" name="id" />
<field column="profile_id" name="profile_id" />
<field column="position_id" name="position_id" />
<field column="position_name" name="position_name" />
<field column="position_name" name="position_name_1" />
<field column="position_name" name="position_name_2" />
<field column="branche_id" name="branche_id" />
<field column="branche_name" name="branche_name" />
<field column="sub_branche_id" name="sub_branche_id" />
<field column="sub_branche_name" name="sub_branche_name" />
<field column="position_class" name="position_class" />
<field column="position_class_name" name="position_class_name" />
<field column="level" name="level" />
<field column="experience" name="experience" />
<field column="grade" name="grade" />
<field column="position_deleted" name="position_deleted" />
<!-- Company -->
<field column="profile_user_id" name="user_id" />
<field column="company_id" name="company_id" />
<field column="profile_active" name="active" />
<field column="profile_visibility" name="visibility" />
<field column="profile_type" name="profile_type" />
<field column="profile_gender" name="gender" />
<field column="profile_initials" name="initials" />
<field column="profile_firstname" name="firstname" />
<field column="profile_infix" name="infix" />
<field column="profile_lastname" name="lastname" />
<field column="profile_email" name="email" />
<field column="profile_zip" name="zip" />
<field column="profile_lat" name="latlng_0_coordinate" />
<field column="profile_lon" name="latlng_1_coordinate" />
<field column="profile_city" name="city" />
<field column="profile_travel_ability" name="travel_ability" />
<field column="profile_status" name="profile_status" />
<field column="profile_active" name="profile_active" />
<field column="profile_deleted" name="profile_deleted" />
<field column="user_deleted" name="user_deleted" />
<!-- Profile -->
<field column="company_name" name="company_name" />
<field column="company_businesstype" name="businesstype" />
<field column="company_amount_of_people" name="amount_of_people" />
<field column="company_visibility" name="company_visibility" />
<field column="company_valid" name="company_valid" />
<field column="company_deleted" name="company_deleted" />
</entity>
</document>
</dataConfig>
