Skip to content

Commit 9901316

Browse files
m-kesselm-kessel
committed
#53 python index support
1 parent 386debb commit 9901316

8 files changed

Lines changed: 416 additions & 18 deletions

File tree

‎core/src/main/java/de/uni_mannheim/swt/lasso/core/model/CodeUnit.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@
3232
*/
3333
public class CodeUnit implements Serializable {
3434

35+
public static final String JAVA = "java";
36+
public static final String PYTHON = "python";
37+
3538
public boolean hasKeyword(String keyword) {
3639
if(unitType == CodeUnitType.METHOD) {
3740
if(CollectionUtils.isEmpty(getMethodSignatureParamsOrderedKeywordsFq())) {
@@ -79,21 +82,39 @@ public void setLql(String lql) {
7982
this.lql = lql;
8083
}
8184

85+
public boolean isJava() {
86+
return StringUtils.equals(getLang(), JAVA);
87+
}
88+
89+
public boolean isPython() {
90+
return StringUtils.equals(getLang(), PYTHON);
91+
}
92+
93+
public String getLang() {
94+
return lang;
95+
}
96+
97+
public void setLang(String lang) {
98+
this.lang = lang;
99+
}
100+
82101
/**
83102
* Code unit.
84103
*
85104
* @author Marcus Kessel
86105
*/
87106
public enum CodeUnitType {
88107
CLASS,
89-
METHOD,
90-
//MODULE // a coherent system of classes (e.g., project)
108+
METHOD, // METHOD OR FUNCTION
109+
MODULE // Python Module
91110
}
92111

93112
private String workerNodeId;
94113

95114
private String id;
96115

116+
private String lang;
117+
97118
private String dataSource;
98119

99120
private String parentId;

‎datasource-maven/src/main/java/de/uni_mannheim/swt/lasso/datasource/maven/MavenCodeUnitUtils.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,25 @@ public static CodeUnit toImplementation(SolrDocument doc) {
4545
CodeUnit i = new CodeUnit();
4646

4747
i.setDocType((String) doc.getFieldValue("doctype_s"));
48-
i.setUnitType("method".equals(i.getDocType()) ? CodeUnit.CodeUnitType.METHOD : CodeUnit.CodeUnitType.CLASS);
48+
49+
switch(i.getDocType()) {
50+
case "method", "function":
51+
i.setUnitType(CodeUnit.CodeUnitType.METHOD);
52+
break;
53+
case "class":
54+
i.setUnitType(CodeUnit.CodeUnitType.CLASS);
55+
break;
56+
case "module":
57+
i.setUnitType(CodeUnit.CodeUnitType.MODULE);
58+
break;
59+
default:
60+
throw new IllegalArgumentException("Unknown docType " + i.getDocType());
61+
}
4962

5063
i.setId((String) doc.getFirstValue("id"));
5164
i.setParentId((String) doc.getFirstValue("pid_s"));
5265
i.setName((String) doc.getFirstValue("name_sexact"));
66+
i.setLang((String) doc.getFirstValue("lang"));
5367
i.setPackagename((String) doc.getFirstValue("packagename_sexact"));
5468
i.setBytecodeName((String) doc.getFirstValue("bytecodename_s"));
5569
i.setGroupId((String) doc.getFirstValue("groupId"));

‎datasource-maven/src/main/java/de/uni_mannheim/swt/lasso/datasource/maven/lsl/MavenQuery.groovy‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ class MavenQuery extends LassoSpec {
6868
*/
6969
//public static String[] CONSTRAINTS = ["doctype_s:class", "type:c", /*"!classifier_s:tests",*/ "{!collapse field=hash nullPolicy=expand sort='versionHead_ti asc'}"]
7070
// XXX disabled collapsing (not needed anymore)
71-
public static String[] CONSTRAINTS = ["doctype_s:class", "type:c"]
71+
public static String[] CONSTRAINTS = ["doctype_s:class", "type:c", "lang:java"]
7272

7373
String query = ''
7474
String queryType = 'class'
@@ -115,13 +115,32 @@ class MavenQuery extends LassoSpec {
115115
options.setFullyQualified(fullyQualified)
116116
}
117117

118+
def unitType(String type) {
119+
if (type.equals("module_or_class")) {
120+
constraints.removeAll({it.startsWith('doctype_s:')})
121+
constraints << "!doctype_s:function" // NOT FUNCTION
122+
} else {
123+
docType(type)
124+
}
125+
126+
// also remove class type (c = class, i = interface etc.)
127+
constraints.removeAll({it.startsWith('type:')})
128+
}
129+
118130
def docType(String type) {
119131
// replace type
120132
constraints.removeAll({it.startsWith('doctype_s:')})
121133
String docType = "doctype_s:${type}"
122134
constraints << docType
123135
}
124136

137+
def lang(String lang) {
138+
// replace lang
139+
constraints.removeAll({it.startsWith('lang:')})
140+
String langType = "lang:${lang}"
141+
constraints << langType
142+
}
143+
125144
def excludeClassesByKeywords(List keywords) {
126145
keywords.each {k ->
127146
filter "-keyword_ss:\"${k}\""

‎datasource-maven/src/main/java/de/uni_mannheim/swt/lasso/index/repo/MavenCentralRepository.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,9 @@ public class MavenCentralRepository extends SolrRepository {
9999
"pid_s",
100100
// Henrik: mcall
101101
"mcall_exact",
102-
"owner"
102+
"owner",
103+
// added for PLs
104+
"lang"
103105
};
104106

105107

datasource-maven/src/test/java/de/uni_mannheim/swt/lasso/index/query/lql/LQL2LuceneIntegrationTest.java renamed to datasource-maven/src/test/java/de/uni_mannheim/swt/lasso/index/query/lql/JavaLQL2LuceneIntegrationTest.java

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
*/
2020
package de.uni_mannheim.swt.lasso.index.query.lql;
2121

22+
import de.uni_mannheim.swt.lasso.core.model.CodeUnit;
2223
import de.uni_mannheim.swt.lasso.core.model.query.QueryResult;
2324
import de.uni_mannheim.swt.lasso.datasource.maven.MavenDataSource;
2425
import de.uni_mannheim.swt.lasso.datasource.maven.systemtests.HttpUtils;
@@ -37,22 +38,22 @@
3738

3839
import java.io.IOException;
3940

40-
import static org.junit.jupiter.api.Assertions.assertNotNull;
41+
import static org.junit.jupiter.api.Assertions.*;
4142

4243
/**
4344
*
4445
* @author Marcus Kessel
4546
*/
46-
public class LQL2LuceneIntegrationTest {
47+
public class JavaLQL2LuceneIntegrationTest {
4748

4849
private static RandomMavenCentralRepository mavenCentralRepository;
4950
private static MavenCentralIndex mavenCentralIndex;
5051

5152
@BeforeAll
5253
public static void before() {
53-
HttpClient client = HttpUtils.createHttpClient("solr", "");
54+
HttpClient client = HttpUtils.createHttpClient("", "");
5455

55-
SolrClient solrClient = new HttpSolrClient.Builder("http://lassohp10.informatik.uni-mannheim.de:8983/solr/mavencentral2023/")
56+
SolrClient solrClient = new HttpSolrClient.Builder("https://odisse.informatik.uni-mannheim.de/solr/mavenCentral2025/")
5657
.withHttpClient(client).build();
5758
mavenCentralRepository = new RandomMavenCentralRepository(solrClient);
5859

@@ -63,27 +64,35 @@ public static void before() {
6364
}
6465

6566
@Test
66-
public void test_Base64_ext() throws IOException {
67+
public void test_Base64_CLASS() throws IOException {
6768
MavenDataSource mavenDataSource = new MavenDataSource(mavenCentralIndex);
6869

6970
MavenQuery mavenQuery = (MavenQuery) mavenDataSource.createQueryModelForLSL();
7071
LassoContext ctx = new LassoContext();
7172
ctx.setLogger(new SimpleLogger());
7273
ctx.register(mavenQuery);
7374
mavenQuery.setDirectly(true);
74-
mavenQuery.queryForClasses("Base64{encode(byte[])->java.lang.String}", "class-ext");
75+
mavenQuery.queryForClasses("Base64{encode(byte[])->java.lang.String}", "class-simple");
7576
mavenQuery.setRows(25);
7677

7778
QueryResult queryResult = mavenDataSource.query(mavenQuery);
7879
queryResult.getImplementations().forEach(implementation -> {
7980
System.out.println("--------");
8081
System.out.println(implementation.toFQName() + " ("+ implementation.getScore()+") " + "=>" + SignatureUtils.create(implementation).toLQL(true));
8182
System.out.println("--------");
83+
84+
assertEquals(CodeUnit.JAVA, implementation.getLang());
85+
assertFalse(implementation.isPython());
86+
assertTrue(implementation.isJava());
87+
88+
assertEquals(CodeUnit.CodeUnitType.CLASS, implementation.getUnitType());
8289
});
90+
91+
assertEquals(mavenQuery.getRows(), queryResult.getImplementations().size());
8392
}
8493

8594
@Test
86-
public void test_Stack_ext() throws IOException {
95+
public void test_Stack_CLASS() throws IOException {
8796
MavenDataSource mavenDataSource = new MavenDataSource(mavenCentralIndex);
8897

8998
MavenQuery mavenQuery = (MavenQuery) mavenDataSource.createQueryModelForLSL();
@@ -97,19 +106,26 @@ public void test_Stack_ext() throws IOException {
97106
" pop()->java.lang.Object\n" +
98107
" peek()->java.lang.Object\n" +
99108
" size()->int\n" +
100-
"}", "class-ext");
109+
"}", "class-simple");
101110
mavenQuery.setRows(25);
102111

103112
QueryResult queryResult = mavenDataSource.query(mavenQuery);
104113
queryResult.getImplementations().forEach(implementation -> {
105114
System.out.println("--------");
106115
System.out.println(implementation.toFQName() + " ("+ implementation.getScore()+") " + "=>" + SignatureUtils.create(implementation).toLQL(true));
107-
System.out.println("--------");
116+
117+
assertEquals(CodeUnit.JAVA, implementation.getLang());
118+
assertFalse(implementation.isPython());
119+
assertTrue(implementation.isJava());
120+
121+
assertEquals(CodeUnit.CodeUnitType.CLASS, implementation.getUnitType());
108122
});
123+
124+
assertEquals(mavenQuery.getRows(), queryResult.getImplementations().size());
109125
}
110126

111127
@Test
112-
public void test_MultiMap_ext() throws IOException {
128+
public void test_MultiMap_CLASS() throws IOException {
113129
MavenDataSource mavenDataSource = new MavenDataSource(mavenCentralIndex);
114130

115131
MavenQuery mavenQuery = (MavenQuery) mavenDataSource.createQueryModelForLSL();
@@ -121,19 +137,27 @@ public void test_MultiMap_ext() throws IOException {
121137
" put(java.lang.Object,java.lang.Object)->java.lang.Object\n" +
122138
" getValues(java.lang.Object)->java.util.List\n" +
123139
" size()->int\n" +
124-
"}", "class-ext");
140+
"}", "class-simple");
125141
mavenQuery.setRows(25);
126142

127143
QueryResult queryResult = mavenDataSource.query(mavenQuery);
128144
queryResult.getImplementations().forEach(implementation -> {
129145
System.out.println("--------");
130146
System.out.println(implementation.toFQName() + " ("+ implementation.getScore()+") " + "=>" + SignatureUtils.create(implementation).toLQL(true));
131147
System.out.println("--------");
148+
149+
assertEquals(CodeUnit.JAVA, implementation.getLang());
150+
assertFalse(implementation.isPython());
151+
assertTrue(implementation.isJava());
152+
153+
assertEquals(CodeUnit.CodeUnitType.CLASS, implementation.getUnitType());
132154
});
155+
156+
assertEquals(mavenQuery.getRows(), queryResult.getImplementations().size());
133157
}
134158

135159
@Test
136-
public void test_StringComparator_ext() throws IOException {
160+
public void test_StringComparator_CLASS() throws IOException {
137161
MavenDataSource mavenDataSource = new MavenDataSource(mavenCentralIndex);
138162

139163
MavenQuery mavenQuery = (MavenQuery) mavenDataSource.createQueryModelForLSL();
@@ -142,14 +166,22 @@ public void test_StringComparator_ext() throws IOException {
142166
ctx.register(mavenQuery);
143167
mavenQuery.setDirectly(true);
144168
mavenQuery.queryForClasses("StringComparator {\n" +
145-
" compare(java.lang.String,java.lang.String)->int}", "class-ext");
169+
" compare(java.lang.String,java.lang.String)->int}", "class-simple");
146170
mavenQuery.setRows(25);
147171

148172
QueryResult queryResult = mavenDataSource.query(mavenQuery);
149173
queryResult.getImplementations().forEach(implementation -> {
150174
System.out.println("--------");
151175
System.out.println(implementation.toFQName() + " ("+ implementation.getScore()+") " + "=>" + SignatureUtils.create(implementation).toLQL(true));
152176
System.out.println("--------");
177+
178+
assertEquals(CodeUnit.JAVA, implementation.getLang());
179+
assertFalse(implementation.isPython());
180+
assertTrue(implementation.isJava());
181+
182+
assertEquals(CodeUnit.CodeUnitType.CLASS, implementation.getUnitType());
153183
});
184+
185+
assertEquals(mavenQuery.getRows(), queryResult.getImplementations().size());
154186
}
155187
}

0 commit comments

Comments
 (0)