#38 added readme for crawler

m-kessel · m-kessel · commit 21494880396b · 2025-05-08T13:57:55.000+02:00
diff --git a/DEPLOY_GITLAB_DOCKER.md b/DEPLOY_GITLAB_DOCKER.md
@@ -41,7 +41,7 @@ docker pull sonatype/nexus3:3.79.0
 docker run -d -p 8081:8081 --name lasso-nexus sonatype/nexus3
 
 # configure
-docker exec -it nexus bash
+docker exec -it lasso-nexus bash
 cat sonatype-work/nexus3/admin.password
 # configure in repos in http://localhost:8081/ (lasso-deploy, lasso-web)
 # see https://softwareobservatorium.github.io/web/docs/infrastructure/nexus
diff --git a/arena/src/main/java/de/uni_mannheim/swt/lasso/arena/search/SolrInstance.java b/arena/src/main/java/de/uni_mannheim/swt/lasso/arena/search/SolrInstance.java
@@ -73,7 +73,7 @@ public static SolrInstance mavenCentral2017() {
     }
 
     public static SolrInstance mavenCentral2023() {
-        return new SolrInstance("mavencentral2023", "", "", "http://lassohp10.informatik.uni-mannheim.de:8983/solr/mavencentral2023/");
+        return new SolrInstance("mavencentral2023", "", "", "https://odisse.informatik.uni-mannheim.de/solr/mavencentral2023/");
     }
 
     public static SolrInstance secorpora2022() {
diff --git a/crawler/README.md b/crawler/README.md
@@ -0,0 +1,25 @@
+# Changes 08.05.25
+
+Issues with automatic download of index (too large to unzip programmatically?)
+
+## Manual Download of Index
+
+see https://maven.apache.org/repository/central-index.html
+
+see https://repo.maven.apache.org/maven2/.index/
+
+download the Central index: nexus-maven-repository-index.gz
+download Maven Indexer CLI and unpack the index to raw Lucene index directory:
+
+```bash
+java -jar indexer-cli-5.1.1.jar --unpack nexus-maven-repository-index.gz --destination central-lucene-index --type full
+```
+
+Results in same directory as created by crawler component
+
+Run crawler
+
+```bash
+wget "http://swtweb.informatik.uni-mannheim.de/nexus/repository/maven-snapshots/de/uni-mannheim/swt/lasso/crawler/1.0.0-SNAPSHOT/crawler-1.0.0-20250508.085922-88.jar" # identify latest snapshot ..
+java -Xms60G -Xmx60G -Dindexer.work.path=lasso_crawler -Dbatch.maven.repo.url=https://repo1.maven.org/maven2/ -Dlasso.indexer.worker.threads=8 -Dbatch.maven.index.update=false -Dbatch.maven.latest.head=1 -jar crawler-1.0.0-20250508.085922-88.jar
+```

Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ public static SolrInstance mavenCentral2017() {`
`73`	`73`	`}`
`74`	`74`
`75`	`75`	`public static SolrInstance mavenCentral2023() {`
`76`		`- return new SolrInstance("mavencentral2023", "", "", "http://lassohp10.informatik.uni-mannheim.de:8983/solr/mavencentral2023/");`
	`76`	`+ return new SolrInstance("mavencentral2023", "", "", "https://odisse.informatik.uni-mannheim.de/solr/mavencentral2023/");`
`77`	`77`	`}`
`78`	`78`
`79`	`79`	`public static SolrInstance secorpora2022() {`