Skip to content

Commit f084780

Browse files
committed
melting-pot: improve local git repository caching
This cuts down on excessive network use. The first time a particular G:A is requested at any version, it gets fully (not shallowly) cloned into the user's melting-pot cache folder. From there, the tag for the given version is cloned shallowly from the locally cached repository. In this way, complex melting pots can be repeatedly generated without hitting the network in most cases. This commit also improves debug and info messages in a few places.
1 parent bff241c commit f084780

1 file changed

Lines changed: 43 additions & 26 deletions

File tree

‎melting-pot.sh‎

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -347,16 +347,18 @@ pom() {
347347
xpath() {
348348
local xmlFile="$1"
349349
shift
350+
local expression="$@"
350351
local xpath="/"
351352
while [ $# -gt 0 ]
352353
do
353354
# NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075
354355
xpath="$xpath/*[local-name()='$1']"
355356
shift
356357
done
357-
debug "xmllint --xpath \"$xpath\" \"$xmlFile\""
358-
xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null |
359-
sed -E 's/^[^>]*>(.*)<[^<]*$/\1/'
358+
local value=$(xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null |
359+
sed -E 's/^[^>]*>(.*)<[^<]*$/\1/')
360+
debug "xpath $xmlFile $expression -> $value"
361+
echo "$value"
360362
}
361363

362364
# For the given GAV ($1), recursively gets the value of the
@@ -399,6 +401,10 @@ scmTag() {
399401
local a=$(artifactId "$1")
400402
local v=$(version "$1")
401403
local scmURL="$(scmURL "$1")"
404+
# TODO: Avoid network use. We can scan the locally cached repo.
405+
# But this gets complicated when the locally cached repo is
406+
# out of date, and the needed tag is not there yet...
407+
debug "git ls-remote --tags \"$scmURL\" | sed 's/.*refs\/tags\///'"
402408
local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' ||
403409
error "$1: Invalid scm url: $scmURL")"
404410
for tag in "$a-$v" "$v" "v$v"
@@ -415,43 +421,54 @@ scmTag() {
415421
fi
416422
}
417423

418-
# Fetches the source code for the given GAV. Returns the directory.
419-
retrieveSource() {
420-
local scmURL="$(scmURL "$1")"
421-
test "$scmURL" || die "Cannot glean SCM URL for $1" 10
422-
local scmBranch
423-
test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")"
424+
# Ensures the source code for the given GAV exists in the melting-pot
425+
# structure, and is up-to-date with the remote. Returns the directory.
426+
resolveSource() {
424427
local g=$(groupId "$1")
425428
local a=$(artifactId "$1")
426-
local v=$(version "$1")
427-
local sourceDir="$meltingPotCache/$g/$a/$v"
428-
if [ -d "$sourceDir" ]
429+
local cachedRepoDir="$meltingPotCache/$g/$a"
430+
if [ ! -d "$cachedRepoDir" ]
429431
then
430-
debug "Using previously fetched project source at $sourceDir"
431-
else
432-
# Source has never been fetched before. Save it into ~/.scijava/melting-pot.
433-
debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$sourceDir\""
434-
git clone "$scmURL" --branch "$scmBranch" --depth 1 "$sourceDir" 2> /dev/null ||
435-
die "Could not fetch project source for $1" 3
432+
# Source does not exist locally. Clone it into the melting pot cache.
433+
local scmURL="$(scmURL "$1")"
434+
test "$scmURL" || die "$1: cannot glean SCM URL" 10
435+
info "$1: cached repository not found; cloning from remote: $scmURL"
436+
debug "git clone --bare \"$scmURL\" \"$cachedRepoDir\""
437+
git clone --bare "$scmURL" "$cachedRepoDir" 2> /dev/null ||
438+
die "$1: could not clone project source from $scmURL" 3
436439
fi
437-
# Copy the pristine cached source directory into the melting-pot structure.
440+
441+
# Check whether the needed branch/tag exists.
442+
local scmBranch
443+
test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")"
444+
debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\trefs/tags/$scmBranch$\""
445+
git ls-remote "file://$cachedRepoDir" | grep -q "\trefs/tags/$scmBranch$" || {
446+
# Couldn't find the scmBranch as a tag in the cached repo. Either the
447+
# tag is new, or it's not a tag ref at all (e.g. it's a branch).
448+
# So let's update from the original remote repository.
449+
info "$1: local tag not found for ref '$scmBranch'; updating cached repository: $cachedRepoDir"
450+
(cd "$cachedRepoDir" && debug "git fetch --tags" && git fetch --tags)
451+
}
452+
453+
# Shallow clone the source at the given version into melting-pot structure.
438454
local destDir="$g/$a"
439-
mkdir -p "$(dirname "$destDir")"
440-
cp -rp "$sourceDir" "$destDir"
455+
debug "git clone \"file://$cachedRepoDir\" --branch \"$scmBranch\" --depth 1 \"$destDir\""
456+
git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null ||
457+
die "$1: could not clone branch '$scmBranch' from local cache" 15
441458

442459
# Now verify that the cloned pom.xml contains the expected version!
443460
local expectedVersion=$(version "$1")
444461
local actualVersion=$(xpath "$destDir/pom.xml" project version)
445462
test "$expectedVersion" = "$actualVersion" ||
446-
die "POM for $1 contains wrong version: $actualVersion" 14
463+
warn "$1: POM contains wrong version: $actualVersion"
447464

448465
echo "$destDir"
449466
}
450467

451468
# Gets the list of dependencies for the project in the CWD.
452469
deps() {
453470
cd "$1"
454-
debug "mvn dependency:list"
471+
debug "mvn -DincludeScope=runtime -B dependency:list"
455472
local depList="$(mvn -DincludeScope=runtime -B dependency:list)" ||
456473
die "Problem fetching dependencies!" 5
457474
echo "$depList" | grep '^\[INFO\] [^ ]' |
@@ -626,7 +643,7 @@ meltDown() {
626643
else
627644
# Treat specified project as a GAV.
628645
info "Fetching project source"
629-
retrieveSource "$1" "$branch"
646+
resolveSource "$1" "$branch"
630647
fi
631648

632649
# Get the project dependencies.
@@ -653,8 +670,8 @@ meltDown() {
653670

654671
if [ "$(isIncluded "$gav")" ]
655672
then
656-
info "$g:$a: fetching source for version $v"
657-
dir="$(retrieveSource "$gav")"
673+
info "$g:$a: resolving source for version $v"
674+
dir="$(resolveSource "$gav")"
658675
fi
659676
done
660677

0 commit comments

Comments
 (0)