Skip to content

Commit a9da77d

Browse files
Camelia-OrcidCamelia-Orcid
committed
Added the code to store deleted items in panoply dw_deleted_items table
The items will be processed by the script that runs daily and cleaned up after the item is deleted from the corresponding view
1 parent d749f3a commit a9da77d

File tree

6 files changed

+202
-3
lines changed

6 files changed

+202
-3
lines changed

‎orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,32 @@
33
import java.math.BigInteger;
44
import java.util.ArrayList;
55
import java.util.List;
6+
import java.util.concurrent.CompletableFuture;
67

8+
import javax.annotation.Resource;
79
import javax.persistence.Query;
810
import javax.persistence.TypedQuery;
911

12+
import org.orcid.utils.panoply.PanoplyDeletedItem;
13+
import org.orcid.utils.panoply.PanoplyRedshiftClient;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
16+
import org.apache.commons.logging.Log;
1017
import org.orcid.persistence.aop.UpdateProfileLastModified;
1118
import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
1219
import org.orcid.persistence.dao.OrgAffiliationRelationDao;
1320
import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity;
21+
import org.springframework.beans.factory.annotation.Value;
1422
import org.springframework.cache.annotation.Cacheable;
1523
import org.springframework.transaction.annotation.Transactional;
1624

1725
public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliationRelationEntity, Long> implements OrgAffiliationRelationDao {
18-
26+
27+
private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class);
28+
29+
@Value("${org.orcid.persistence.panoply.cleanup.production:false}")
30+
private boolean enablePanoplyCleanupInProduction;
31+
1932
private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION";
2033

2134
private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION";
@@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliation
2942
private static final String AFFILIATION_TYPE_QUALIFICATION = "QUALIFICATION";
3043

3144
private static final String AFFILIATION_TYPE_SERVICE = "SERVICE";
45+
46+
private static final String DW_PANOPLY_AFFILIATION_TABLE = "dw_org_affiliation_relation";
47+
48+
@Resource
49+
private PanoplyRedshiftClient panoplyClient;
3250

3351
public OrgAffiliationRelationDaoImpl() {
3452
super(OrgAffiliationRelationEntity.class);
@@ -51,7 +69,34 @@ public boolean removeOrgAffiliationRelation(String userOrcid, Long orgAffiliatio
5169
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid=:userOrcid and id=:orgAffiliationRelationId");
5270
query.setParameter("userOrcid", userOrcid);
5371
query.setParameter("orgAffiliationRelationId", orgAffiliationRelationId);
54-
return query.executeUpdate() > 0 ? true : false;
72+
if(query.executeUpdate() > 0) {
73+
if(enablePanoplyCleanupInProduction) {
74+
PanoplyDeletedItem item = new PanoplyDeletedItem();
75+
item.setItemId(orgAffiliationRelationId);
76+
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
77+
storeDeletedItemInPanoply(item);
78+
}
79+
return true;
80+
}
81+
return false;
82+
}
83+
84+
private void storeDeletedItemInPanoply(PanoplyDeletedItem item) {
85+
//Store the deleted item in panoply Db without blocking
86+
CompletableFuture.supplyAsync(() -> {
87+
try {
88+
panoplyClient.addPanoplyDeletedItem(item);
89+
return true;
90+
} catch (Exception e) {
91+
LOG.error("Cannot store deleted affiliation in panoply ", e);
92+
return false;
93+
}
94+
}).thenAccept(result -> {
95+
if(! result) {
96+
LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result);
97+
}
98+
99+
});
55100
}
56101

57102
/**
@@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) {
196241
Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId");
197242
query.setParameter("clientSourceId", clientSourceId);
198243
query.executeUpdate();
244+
if(query.executeUpdate() > 0) {
245+
if(enablePanoplyCleanupInProduction) {
246+
PanoplyDeletedItem item = new PanoplyDeletedItem();
247+
item.setClientSourceId(clientSourceId);
248+
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
249+
storeDeletedItemInPanoply(item);
250+
}
251+
}
199252
}
200253

201254
@Override
@@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) {
281334
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid");
282335
query.setParameter("orcid", orcid);
283336
query.executeUpdate();
337+
if(query.executeUpdate() > 0) {
338+
if(enablePanoplyCleanupInProduction) {
339+
PanoplyDeletedItem item = new PanoplyDeletedItem();
340+
item.setOrcid(orcid);
341+
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
342+
storeDeletedItemInPanoply(item);
343+
}
344+
}
345+
284346
}
285347

286348
@Override

‎orcid-persistence/src/main/resources/orcid-persistence-context.xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,22 @@
457457
<property name="entityManager" ref="entityManagerReadOnly" />
458458
</bean>
459459

460+
461+
<!-- Panoply Redshift -->
462+
<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
463+
<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver}" />
464+
<property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
465+
<property name="username" value="\${org.orcid.core.utils.panoply.username}" />
466+
<property name="password" value="\${org.orcid.core.utils.panoply.password}" />
467+
<property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize}" />
468+
<property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout}" />
469+
<property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout}" />
470+
</bean>
471+
472+
<!-- Panoply JdbcTemplate Bean Definition -->
473+
<bean id="panoplyJdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
474+
<property name="dataSource" ref="panoplyRedshiftDataSource" />
475+
</bean>
460476

461477
<!-- Statistics -->
462478
<bean id="statisticsDao" class="org.orcid.persistence.dao.impl.StatisticsDaoImpl">

‎orcid-utils/pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@
4343
<groupId>org.springframework</groupId>
4444
<artifactId>spring-context</artifactId>
4545
</dependency>
46+
<dependency>
47+
<groupId>org.springframework</groupId>
48+
<artifactId>spring-jdbc</artifactId>
49+
</dependency>
4650
<dependency>
4751
<groupId>org.apache.commons</groupId>
4852
<artifactId>commons-lang3</artifactId>
@@ -113,6 +117,22 @@
113117
<artifactId>jsoup</artifactId>
114118
<version>1.15.4</version>
115119
</dependency>
120+
121+
<!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 -->
122+
<dependency>
123+
<groupId>com.amazon.redshift</groupId>
124+
<artifactId>redshift-jdbc42</artifactId>
125+
<version>2.1.0.30</version>
126+
</dependency>
127+
128+
129+
<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
130+
<dependency>
131+
<groupId>com.zaxxer</groupId>
132+
<artifactId>HikariCP</artifactId>
133+
<version>5.0.1</version>
134+
</dependency>
135+
116136

117137
<!-- Test dependencies -->
118138

‎orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package org.orcid.utils.panoply;
2+
3+
public class PanoplyDeletedItem {
4+
private Long id;
5+
private String dwTable;
6+
private Long itemId;
7+
private String clientSourceId;
8+
private String orcid;
9+
10+
public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation";
11+
public final String DW_WORK = "dw_work";
12+
13+
public Long getId() {
14+
return id;
15+
}
16+
public void setId(Long id) {
17+
this.id = id;
18+
}
19+
public String getDwTable() {
20+
return dwTable;
21+
}
22+
public void setDwTable(String dwTable) {
23+
this.dwTable = dwTable;
24+
}
25+
public Long getItemId() {
26+
return itemId;
27+
}
28+
public void setItemId(Long itemId) {
29+
this.itemId = itemId;
30+
}
31+
public String getClientSourceId() {
32+
return clientSourceId;
33+
}
34+
public void setClientSourceId(String clientSourceId) {
35+
this.clientSourceId = clientSourceId;
36+
}
37+
public String getOrcid() {
38+
return orcid;
39+
}
40+
public void setOrcid(String orcid) {
41+
this.orcid = orcid;
42+
}
43+
44+
@Override
45+
public String toString() {
46+
return "PanoplyDeletedItem{" +
47+
"id=" + id +
48+
", dwTable='" + dwTable + '\'' +
49+
", itemId='" + itemId + '\'' +
50+
", clientSourceId='" + clientSourceId + '\'' +
51+
", orcid='" + orcid + '\'' +
52+
'}';
53+
}
54+
55+
}

‎orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package org.orcid.utils.panoply;
2+
3+
import org.springframework.beans.factory.annotation.Autowired;
4+
import org.springframework.beans.factory.annotation.Qualifier;
5+
import org.springframework.beans.factory.annotation.Value;
6+
import org.springframework.context.annotation.Bean;
7+
import org.springframework.jdbc.core.JdbcTemplate;
8+
import org.springframework.stereotype.Repository;
9+
import org.springframework.stereotype.Service;
10+
11+
12+
import java.util.Date;
13+
14+
import javax.annotation.Resource;
15+
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
18+
import org.orcid.utils.alerting.SlackManager;
19+
20+
@Repository
21+
public class PanoplyRedshiftClient {
22+
23+
private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class);
24+
25+
@Autowired
26+
@Qualifier("panoplyJdbcTemplate")
27+
private JdbcTemplate panoplyJdbcTemplate;
28+
29+
30+
public int addPanoplyDeletedItem(PanoplyDeletedItem item) {
31+
LOG.debug("Adding deleted item to panoply DB: " + item.toString());
32+
String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)";
33+
return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable());
34+
}
35+
36+
}

‎properties/development.properties‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true
258258
org.orcid.core.autospam.slackChannel=collab-spam-reports
259259
org.orcid.core.autospam.webhookUrl=
260260

261-
#org.orcid.persistence.liquibase.enabled=false
261+
org.orcid.persistence.liquibase.enabled=false
262262
org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr
263+
264+
org.orcid.persistence.panoply.cleanup.production=false
265+
# Panoply redshift database
266+
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
267+
org.orcid.core.utils.panoply.maxPoolSize=20
268+
org.orcid.core.utils.panoply.password=xxx
269+
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
270+
org.orcid.core.utils.panoply.connectionTimeout=36000
271+
org.orcid.core.utils.panoply.jdbcUrl=xxx
272+
org.orcid.core.utils.panoply.username=xxx

0 commit comments

Comments
 (0)