Skip to content

Commit 558cf7e

Browse files
tgamblintgamblin
committed
spec flatten, normalize, validate; package validate
New operations for manipulating spec and package DAGs. For specs: flatten: gather all deps to the root normalize: Merge constraints and make spec match package DAG For packages: validate_dependencies: Make sure spec constraints in package DAG are sane. Added tests for above methods. Also added beginnings of concretization logic, to turn abstract spec into a concrete one. Still need proper tests for normalize().
1 parent db07c7f commit 558cf7e

File tree

21 files changed

+641
-167
lines changed

21 files changed

+641
-167
lines changed

‎lib/spack/spack/cmd/test.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,13 @@ def setup_parser(subparser):
1919

2020
def test(parser, args):
2121
if args.all:
22-
for name in list_modules(spack.test_path):
22+
for name in list_modules(spack.test_path, directories=False):
2323
print "Running Tests: %s" % name
2424
spack.test.run(name, verbose=args.verbose)
2525

2626
elif not args.names:
2727
print "Available tests:"
28-
colify(list_modules(spack.test_path))
29-
28+
colify(list_modules(spack.test_path, directories=False))
3029

3130
else:
3231
for name in args.names:

‎lib/spack/spack/compilers/__init__.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,7 @@ def supported_compilers():
1111
return [c for c in list_modules(spack.compilers_path)]
1212

1313

14-
def get_compiler():
15-
return Compiler('gcc', spack.compilers.gcc.get_version())
14+
@memoized
15+
def default_compiler():
16+
from spack.spec import Compiler
17+
return Compiler('gcc', gcc.get_version())

‎lib/spack/spack/concretize.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""
2+
Functions here are used to take abstract specs and make them concrete.
3+
For example, if a spec asks for a version between 1.8 and 1.9, these
4+
functions might take will take the most recent 1.9 version of the
5+
package available. Or, if the user didn't specify a compiler for a
6+
spec, then this will assign a compiler to the spec based on defaults
7+
or user preferences.
8+
9+
TODO: make this customizable and allow users to configure
10+
concretization policies.
11+
"""
12+
import spack.arch
13+
import spack.compilers
14+
from spack.version import *
15+
from spack.spec import *
16+
17+
18+
def concretize_version(spec):
19+
"""If the spec is already concrete, return. Otherwise take
20+
the most recent available version, and default to the package's
21+
version if there are no avaialble versions.
22+
"""
23+
# return if already concrete.
24+
if spec.versions.concrete:
25+
return
26+
27+
pkg = speck.package
28+
available = pkg.available_versions
29+
30+
# If there are known avaialble versions, return the most recent
31+
if versions:
32+
spec.versions = ver([avaialble[-1]])
33+
else:
34+
spec.versions = ver([pkg.version])
35+
36+
37+
def concretize_architecture(spec):
38+
"""If the spec already had an architecture, return. Otherwise if
39+
the root of the DAG has an architecture, then use that.
40+
Otherwise take the system's default architecture.
41+
42+
Intuition: Architectures won't be set a lot, and generally you
43+
want the host system's architecture. When architectures are
44+
mised in a spec, it is likely because the tool requries a
45+
cross-compiled component, e.g. for tools that run on BlueGene
46+
or Cray machines. These constraints will likely come directly
47+
from packages, so require the user to be explicit if they want
48+
to mess with the architecture, and revert to the default when
49+
they're not explicit.
50+
"""
51+
if spec.architecture is not None:
52+
return
53+
54+
if spec.root.architecture:
55+
spec.architecture = spec.root.architecture
56+
else:
57+
spec.architecture = spack.arch.sys_type()
58+
59+
60+
def concretize_compiler(spec):
61+
"""Currently just sets the compiler to gcc or throws an exception
62+
if the compiler is set to something else.
63+
64+
TODO: implement below description.
65+
66+
If the spec already has a compiler, we're done. If not, then
67+
take the compiler used for the nearest ancestor with a concrete
68+
compiler, or use the system default if there is no ancestor
69+
with a compiler.
70+
71+
Intuition: Use the system default if no package that depends on
72+
this one has a strict compiler requirement. Otherwise, try to
73+
build with the compiler that will be used by libraries that
74+
link to this one, to maximize compatibility.
75+
"""
76+
if spec.compiler.concrete:
77+
if spec.compiler != spack.compilers.default_compiler():
78+
raise spack.spec.UnknownCompilerError(str(spec.compiler))
79+
else:
80+
spec.compiler = spack.compilers.default_compiler()

‎lib/spack/spack/package.py‎

Lines changed: 57 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,13 @@
1818

1919
from spack import *
2020
import spack.spec
21+
import spack.error
2122
import packages
2223
import tty
2324
import attr
2425
import validate
2526
import url
2627

27-
from spec import Compiler
2828
from version import *
2929
from multi_function import platform
3030
from stage import Stage
@@ -249,7 +249,7 @@ class SomePackage(Package):
249249
# These variables are per-package metadata will be defined by subclasses.
250250
#
251251
"""By default a package has no dependencies."""
252-
dependencies = []
252+
dependencies = {}
253253

254254
#
255255
# These are default values for instance variables.
@@ -371,21 +371,51 @@ def dependents(self):
371371
return tuple(self._dependents)
372372

373373

374-
def sanity_check(self):
375-
"""Ensure that this package and its dependencies don't have conflicting
376-
requirements."""
377-
deps = sorted(self.all_dependencies, key=lambda d: d.name)
374+
def preorder_traversal(self, visited=None):
375+
if visited is None:
376+
visited = set()
377+
378+
if self.name in visited:
379+
return
380+
visited.add(self.name)
381+
382+
yield self
383+
for name, spec in self.dependencies.iteritems():
384+
for pkg in packages.get(name).preorder_traversal(visited):
385+
yield pkg
378386

379387

388+
def validate_dependencies(self):
389+
"""Ensure that this package and its dependencies all have consistent
390+
constraints on them.
391+
"""
392+
# This algorithm just attempts to merge all the constraints on the same
393+
# package together, loses information about the source of the conflict.
394+
# What we'd really like to know is exactly which two constraints
395+
# conflict, but that algorithm is more expensive, so we'll do it
396+
# the simple, less informative way for now.
397+
merged = spack.spec.DependencyMap()
398+
399+
try:
400+
for pkg in self.preorder_traversal():
401+
for name, spec in pkg.dependencies.iteritems():
402+
if name not in merged:
403+
merged[name] = spec.copy()
404+
else:
405+
merged[name].constrain(spec)
406+
407+
except spack.spec.UnsatisfiableSpecError, e:
408+
raise InvalidPackageDependencyError(
409+
"Package %s has inconsistent dependency constraints: %s"
410+
% (self.name, e.message))
411+
380412

381413
@property
382414
@memoized
383415
def all_dependencies(self):
384416
"""Dict(str -> Package) of all transitive dependencies of this package."""
385-
all_deps = set(self.dependencies)
386-
for dep in self.dependencies:
387-
dep_pkg = packages.get(dep.name)
388-
all_deps = all_deps.union(dep_pkg.all_dependencies)
417+
all_deps = {name : dep for dep in self.preorder_traversal}
418+
del all_deps[self.name]
389419
return all_deps
390420

391421

@@ -533,7 +563,7 @@ def setup_install_environment(self):
533563

534564
# Pass along prefixes of dependencies here
535565
path_set(SPACK_DEPENDENCIES,
536-
[dep.package.prefix for dep in self.dependencies])
566+
[dep.package.prefix for dep in self.dependencies.values()])
537567

538568
# Install location
539569
os.environ[SPACK_PREFIX] = self.prefix
@@ -544,7 +574,7 @@ def setup_install_environment(self):
544574

545575
def do_install_dependencies(self):
546576
# Pass along paths of dependencies here
547-
for dep in self.dependencies:
577+
for dep in self.dependencies.values():
548578
dep.package.do_install()
549579

550580

@@ -607,7 +637,7 @@ def do_clean_dist(self):
607637
@property
608638
def available_versions(self):
609639
if not self._available_versions:
610-
self._available_versions = VersionList()
640+
self._available_versions = ver([self.version])
611641
try:
612642
# Run curl but grab the mime type from the http headers
613643
listing = spack.curl('-s', '-L', self.list_url, return_output=True)
@@ -617,18 +647,18 @@ def available_versions(self):
617647
for s in strings:
618648
match = re.search(wildcard, s)
619649
if match:
620-
self._available_versions.add(ver(match.group(0)))
650+
self._available_versions.add(Version(match.group(0)))
651+
652+
if not self._available_versions:
653+
tty.warn("Found no versions for %s" % self.name,
654+
"Packate.available_versions may require adding the list_url attribute",
655+
"to the package to tell Spack where to look for versions.")
621656

622-
except CalledProcessError:
657+
except subprocess.CalledProcessError:
623658
tty.warn("Fetching %s failed." % self.list_url,
624659
"Package.available_versions requires an internet connection.",
625660
"Version list may be incomplete.")
626661

627-
if not self._available_versions:
628-
tty.warn("Found no versions for %s" % self.name,
629-
"Packate.available_versions may require adding the list_url attribute",
630-
"to the package to tell Spack where to look for versions.")
631-
self._available_versions = [self.version]
632662
return self._available_versions
633663

634664

@@ -654,3 +684,10 @@ def __call__(self, *args, **kwargs):
654684
args = (jobs,) + args
655685

656686
super(MakeExecutable, self).__call__(*args, **kwargs)
687+
688+
689+
class InvalidPackageDependencyError(spack.error.SpackError):
690+
"""Raised when package specification is inconsistent with requirements of
691+
its dependencies."""
692+
def __init__(self, message):
693+
super(InvalidPackageDependencyError, self).__init__(message)

‎lib/spack/spack/packages/__init__.py‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
instances = {}
2222

23+
2324
def get(pkg_name):
2425
if not pkg_name in instances:
2526
package_class = get_class_for_package_name(pkg_name)
@@ -85,9 +86,18 @@ def get_class_for_package_name(pkg_name):
8586
else:
8687
raise UnknownPackageError(pkg_name)
8788

89+
# Figure out pacakges module from spack.packages_path
90+
# This allows us to change the module path.
91+
if not re.match(r'%s' % spack.module_path, spack.packages_path):
92+
raise RuntimeError("Packages path is not a submodule of spack.")
93+
94+
# TODO: replace this with a proper package DB class, instead of this hackiness.
95+
packages_path = re.sub(spack.module_path + '\/+', 'spack.', spack.packages_path)
96+
packages_module = re.sub(r'\/', '.', packages_path)
97+
8898
class_name = pkg_name.capitalize()
8999
try:
90-
module_name = "%s.%s" % (__name__, pkg_name)
100+
module_name = "%s.%s" % (packages_module, pkg_name)
91101
module = __import__(module_name, fromlist=[class_name])
92102
except ImportError, e:
93103
tty.die("Error while importing %s.%s:\n%s" % (pkg_name, class_name, e.message))
@@ -107,8 +117,8 @@ def compute_dependents():
107117
if pkg._dependents is None:
108118
pkg._dependents = []
109119

110-
for dep in pkg.dependencies:
111-
dpkg = get(dep.name)
120+
for name, dep in pkg.dependencies.iteritems():
121+
dpkg = get(name)
112122
if dpkg._dependents is None:
113123
dpkg._dependents = []
114124
dpkg._dependents.append(pkg.name)
@@ -130,8 +140,8 @@ def quote(string):
130140
deps = []
131141
for pkg in all_packages():
132142
out.write(' %-30s [label="%s"]\n' % (quote(pkg.name), pkg.name))
133-
for dep in pkg.dependencies:
134-
deps.append((pkg.name, dep.name))
143+
for dep_name, dep in pkg.dependencies.iteritems():
144+
deps.append((pkg.name, dep_name))
135145
out.write('\n')
136146

137147
for pair in deps:

‎lib/spack/spack/packages/libdwarf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class Libdwarf(Package):
1111

1212
list_url = "http://reality.sgiweb.org/davea/dwarf.html"
1313

14-
depends_on("libelf@0:1")
14+
depends_on("libelf")
1515

1616

1717
def clean(self):

‎lib/spack/spack/parse.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,12 +91,16 @@ def expect(self, id):
9191
self.next_token_error("Unexpected end of input")
9292
sys.exit(1)
9393

94-
def parse(self, text):
94+
def setup(self, text):
9595
self.text = text
9696
self.push_tokens(self.lexer.lex(text))
97+
98+
def parse(self, text):
99+
self.setup(text)
97100
return self.do_parse()
98101

99102

103+
100104
class ParseError(spack.error.SpackError):
101105
"""Raised when we don't hit an error while parsing."""
102106
def __init__(self, message, string, pos):

‎lib/spack/spack/relations.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ def depends_on(*specs):
5454
"""
5555
# Get the enclosing package's scope and add deps to it.
5656
locals = sys._getframe(1).f_locals
57-
dependencies = locals.setdefault("dependencies", [])
57+
dependencies = locals.setdefault("dependencies", {})
5858
for string in specs:
5959
for spec in spack.spec.parse(string):
60-
dependencies.append(spec)
60+
dependencies[spec.name] = spec
6161

6262

6363
def provides(*args):

0 commit comments

Comments
 (0)