"""Error thrown when fetching the cache failed, usually a composite error list."""

def __init__(self, errors):

if not isinstance(errors, list):

raise TypeError("Expected a list of errors")

self.errors = errors

if len(errors) > 1:

msg = " Error {0}: {1}: {2}"

self.message = "Multiple errors during fetching:\n"

self.message += "\n".join((

msg.format(i + 1, err.__class__.__name__, str(err))

for (i, err) in enumerate(errors)

))

else:

err = errors[0]

self.message = "{0}: {1}".format(err.__class__.__name__, str(err))

"""

def __init__(self, cache_root):

self._index_cache_root = cache_root

# the key associated with the serialized _local_index_cache

self._index_contents_key = 'contents.json'

# a FileCache instance storing copies of remote binary cache indices

self._index_file_cache = None

# stores a map of mirror URL to index hash and cache key (index path)

self._local_index_cache = None

# hashes of remote indices already ingested into the concrete spec

# cache (_mirrors_for_spec)

self._specs_already_associated = set()

# _mirrors_for_spec is a dictionary mapping DAG hashes to lists of

# entries indicating mirrors where that concrete spec can be found.

# Each entry is a dictionary consisting of:

#

# - the mirror where the spec is, keyed by ``mirror_url``

# - the concrete spec itself, keyed by ``spec`` (including the

# full hash, since the dag hash may match but we want to

# use the updated source if available)

self._mirrors_for_spec = {}

def _init_local_index_cache(self):

if not self._index_file_cache:

self._index_file_cache = file_cache.FileCache(

self._index_cache_root)

cache_key = self._index_contents_key

self._index_file_cache.init_entry(cache_key)

cache_path = self._index_file_cache.cache_path(cache_key)

self._local_index_cache = {}

if os.path.isfile(cache_path):

with self._index_file_cache.read_transaction(

cache_key) as cache_file:

self._local_index_cache = json.load(cache_file)

def clear(self):

""" For testing purposes we need to be able to empty the cache and

if self._index_file_cache:

self._index_file_cache.destroy()

self._index_file_cache = None

self._local_index_cache = None

self._specs_already_associated = set()

self._mirrors_for_spec = {}

def _write_local_index_cache(self):

self._init_local_index_cache()

cache_key = self._index_contents_key

with self._index_file_cache.write_transaction(cache_key) as (old, new):

json.dump(self._local_index_cache, new)

def regenerate_spec_cache(self, clear_existing=False):

""" Populate the local cache of concrete specs (``_mirrors_for_spec``)

self._init_local_index_cache()

if clear_existing:

self._specs_already_associated = set()

self._mirrors_for_spec = {}

for mirror_url in self._local_index_cache:

cache_entry = self._local_index_cache[mirror_url]

cached_index_path = cache_entry['index_path']

cached_index_hash = cache_entry['index_hash']

if cached_index_hash not in self._specs_already_associated:

self._associate_built_specs_with_mirror(cached_index_path,

mirror_url)

self._specs_already_associated.add(cached_index_hash)

def _associate_built_specs_with_mirror(self, cache_key, mirror_url):

tmpdir = tempfile.mkdtemp()

try:

db_root_dir = os.path.join(tmpdir, 'db_root')

db = spack_db.Database(None, db_dir=db_root_dir,

enable_transaction_locking=False)

self._index_file_cache.init_entry(cache_key)

cache_path = self._index_file_cache.cache_path(cache_key)

with self._index_file_cache.read_transaction(cache_key):

db._read_from_file(cache_path)

spec_list = db.query_local(installed=False, in_buildcache=True)

for indexed_spec in spec_list:

dag_hash = indexed_spec.dag_hash()

full_hash = indexed_spec._full_hash

if dag_hash not in self._mirrors_for_spec:

self._mirrors_for_spec[dag_hash] = []

for entry in self._mirrors_for_spec[dag_hash]:

# A binary mirror can only have one spec per DAG hash, so

# if we already have an entry under this DAG hash for this

# mirror url, we may need to replace the spec associated

# with it (but only if it has a different full_hash).

if entry['mirror_url'] == mirror_url:

if full_hash and full_hash != entry['spec']._full_hash:

entry['spec'] = indexed_spec

break

else:

self._mirrors_for_spec[dag_hash].append({

"mirror_url": mirror_url,

"spec": indexed_spec,

})

finally:

shutil.rmtree(tmpdir)

def get_all_built_specs(self):

spec_list = []

for dag_hash in self._mirrors_for_spec:

# in the absence of further information, all concrete specs

# with the same DAG hash are equivalent, so we can just

# return the first one in the list.

if len(self._mirrors_for_spec[dag_hash]) > 0:

spec_list.append(self._mirrors_for_spec[dag_hash][0]['spec'])

return spec_list

def find_built_spec(self, spec):

"""Look in our cache for the built spec corresponding to ``spec``.

self.regenerate_spec_cache()

return self.find_by_hash(spec.dag_hash())

def find_by_hash(self, find_hash):

"""Same as find_built_spec but uses the hash of a spec.

if find_hash not in self._mirrors_for_spec:

return None

return self._mirrors_for_spec[find_hash]

def update_spec(self, spec, found_list):

"""

spec_dag_hash = spec.dag_hash()

if spec_dag_hash not in self._mirrors_for_spec:

self._mirrors_for_spec[spec_dag_hash] = found_list

else:

current_list = self._mirrors_for_spec[spec_dag_hash]

for new_entry in found_list:

for cur_entry in current_list:

if new_entry['mirror_url'] == cur_entry['mirror_url']:

cur_entry['spec'] = new_entry['spec']

break

else:

current_list.append = {

'mirror_url': new_entry['mirror_url'],

'spec': new_entry['spec'],

}

def update(self):

""" Make sure local cache of buildcache index files is up to date.

self._init_local_index_cache()

mirrors = spack.mirror.MirrorCollection()

configured_mirror_urls = [m.fetch_url for m in mirrors.values()]

items_to_remove = []

spec_cache_clear_needed = False

spec_cache_regenerate_needed = not self._mirrors_for_spec

# First compare the mirror urls currently present in the cache to the

# configured mirrors. If we have a cached index for a mirror which is

# no longer configured, we should remove it from the cache. For any

# cached indices corresponding to currently configured mirrors, we need

# to check if the cache is still good, or needs to be updated.

# Finally, if there are configured mirrors for which we don't have a

# cache entry, we need to fetch and cache the indices from those

# mirrors.

# If, during this process, we find that any mirrors for which we

# already have entries have either been removed, or their index

# hash has changed, then our concrete spec cache (_mirrors_for_spec)

# likely has entries that need to be removed, so we will clear it

# and regenerate that data structure.

# If, during this process, we find that there are new mirrors for

# which do not yet have an entry in our index cache, then we simply

# need to regenerate the concrete spec cache, but do not need to

# clear it first.

# Otherwise the concrete spec cache should not need to be updated at

# all.

fetch_errors = []

all_methods_failed = True

for cached_mirror_url in self._local_index_cache:

cache_entry = self._local_index_cache[cached_mirror_url]

cached_index_hash = cache_entry['index_hash']

cached_index_path = cache_entry['index_path']

if cached_mirror_url in configured_mirror_urls:

# May need to fetch the index and update the local caches

try:

needs_regen = self._fetch_and_cache_index(

cached_mirror_url, expect_hash=cached_index_hash)

all_methods_failed = False

except FetchCacheError as fetch_error:

needs_regen = False

fetch_errors.extend(fetch_error.errors)

# The need to regenerate implies a need to clear as well.

spec_cache_clear_needed |= needs_regen

spec_cache_regenerate_needed |= needs_regen

else:

# No longer have this mirror, cached index should be removed

items_to_remove.append({

'url': cached_mirror_url,

'cache_key': os.path.join(self._index_cache_root,

cached_index_path)

})

spec_cache_clear_needed = True

spec_cache_regenerate_needed = True

# Clean up items to be removed, identified above

for item in items_to_remove:

url = item['url']

cache_key = item['cache_key']

self._index_file_cache.remove(cache_key)

del self._local_index_cache[url]

# Iterate the configured mirrors now. Any mirror urls we do not

# already have in our cache must be fetched, stored, and represented

# locally.

for mirror_url in configured_mirror_urls:

if mirror_url not in self._local_index_cache:

# Need to fetch the index and update the local caches

try:

needs_regen = self._fetch_and_cache_index(mirror_url)

all_methods_failed = False

except FetchCacheError as fetch_error:

fetch_errors.extend(fetch_error.errors)

needs_regen = False

# Generally speaking, a new mirror wouldn't imply the need to

# clear the spec cache, so leave it as is.

if needs_regen:

spec_cache_regenerate_needed = True

self._write_local_index_cache()

if all_methods_failed:

raise FetchCacheError(fetch_errors)

elif spec_cache_regenerate_needed:

self.regenerate_spec_cache(clear_existing=spec_cache_clear_needed)

def _fetch_and_cache_index(self, mirror_url, expect_hash=None):

""" Fetch a buildcache index file from a remote mirror and cache it.

index_fetch_url = url_util.join(

mirror_url, _build_cache_relative_path, 'index.json')

hash_fetch_url = url_util.join(

mirror_url, _build_cache_relative_path, 'index.json.hash')

old_cache_key = None

fetched_hash = None

errors = []

# Fetch the hash first so we can check if we actually need to fetch

# the index itself.

try:

_, _, fs = web_util.read_from_url(hash_fetch_url)

fetched_hash = codecs.getreader('utf-8')(fs).read()

except (URLError, web_util.SpackWebError) as url_err:

errors.append(

RuntimeError("Unable to read index hash {0} due to {1}: {2}".format(

hash_fetch_url, url_err.__class__.__name__, str(url_err)

))

)

# The only case where we'll skip attempting to fetch the buildcache

# index from the mirror is when we already have a hash for this

# mirror, we were able to retrieve one from the mirror, and

# the two hashes are the same.

if expect_hash and fetched_hash:

if fetched_hash == expect_hash:

tty.debug('Cached index for {0} already up to date'.format(

mirror_url))

return False

else:

# We expected a hash, we fetched a hash, and they were not the

# same. If we end up fetching an index successfully and

# replacing our entry for this mirror, we should clean up the

# existing cache file

if mirror_url in self._local_index_cache:

existing_entry = self._local_index_cache[mirror_url]

old_cache_key = existing_entry['index_path']

tty.debug('Fetching index from {0}'.format(index_fetch_url))

# Fetch index itself

try:

_, _, fs = web_util.read_from_url(index_fetch_url)

index_object_str = codecs.getreader('utf-8')(fs).read()

except (URLError, web_util.SpackWebError) as url_err:

errors.append(

RuntimeError("Unable to read index {0} due to {1}: {2}".format(

index_fetch_url, url_err.__class__.__name__, str(url_err)

))

)

raise FetchCacheError(errors)

locally_computed_hash = compute_hash(index_object_str)

if fetched_hash is not None and locally_computed_hash != fetched_hash:

msg = ('Computed hash ({0}) did not match remote ({1}), '

'indicating error in index transmission').format(

locally_computed_hash, expect_hash)

errors.append(RuntimeError(msg))

# We somehow got an index that doesn't match the remote one, maybe

# the next time we try we'll be successful.

raise FetchCacheError(errors)

url_hash = compute_hash(mirror_url)

cache_key = '{0}_{1}.json'.format(

url_hash[:10], locally_computed_hash[:10])

self._index_file_cache.init_entry(cache_key)

with self._index_file_cache.write_transaction(cache_key) as (old, new):

new.write(index_object_str)

self._local_index_cache[mirror_url] = {

'index_hash': locally_computed_hash,

'index_path': cache_key,

}

# clean up the old cache_key if necessary

if old_cache_key:

self._index_file_cache.remove(old_cache_key)

# We fetched an index and updated the local index cache, we should

# regenerate the spec cache as a result.

"""

def __init__(self, file_path):

err_msg = "\n%s\nexists\n" % file_path

err_msg += "Use -f option to overwrite."

"""

def __init__(self, msg):

"""

def __init__(self, msg):

"""

def __init__(self, keys):

err_msg = "Multiple keys available for signing\n%s\n" % keys

err_msg += "Use spack buildcache create -k <key hash> to pick a key."

"""

"""

"""

def __init__(self, msg):

"""

name = os.path.join(prefix, ".spack/binary_distribution")

"""

filename = buildinfo_file_name(prefix)

with open(filename, 'r') as inputfile:

content = inputfile.read()

buildinfo = yaml.load(content)

"""

data = {"text_to_relocate": [], "binary_to_relocate": [],

"link_to_relocate": [], "other": [],

"binary_to_relocate_fullpath": []}

blacklist = (".spack", "man")

# Do this at during tarball creation to save time when tarball unpacked.

# Used by make_package_relative to determine binaries to change.

for root, dirs, files in os.walk(spec.prefix, topdown=True):

dirs[:] = [d for d in dirs if d not in blacklist]

# Directories may need to be relocated too.

for directory in dirs:

dir_path_name = os.path.join(root, directory)

rel_path_name = os.path.relpath(dir_path_name, spec.prefix)

if os.path.islink(dir_path_name):

link = os.readlink(dir_path_name)

if os.path.isabs(link) and link.startswith(spack.store.layout.root):

data['link_to_relocate'].append(rel_path_name)

for filename in files:

path_name = os.path.join(root, filename)

m_type, m_subtype = relocate.mime_type(path_name)

rel_path_name = os.path.relpath(path_name, spec.prefix)

added = False

if os.path.islink(path_name):

link = os.readlink(path_name)

if os.path.isabs(link):

# Relocate absolute links into the spack tree

if link.startswith(spack.store.layout.root):

data['link_to_relocate'].append(rel_path_name)

added = True

if relocate.needs_binary_relocation(m_type, m_subtype):

if ((m_subtype in ('x-executable', 'x-sharedlib', 'x-pie-executable')

and sys.platform != 'darwin') or

(m_subtype in ('x-mach-binary')

and sys.platform == 'darwin') or

(not filename.endswith('.o'))):

data['binary_to_relocate'].append(rel_path_name)

data['binary_to_relocate_fullpath'].append(path_name)

added = True

if relocate.needs_text_relocation(m_type, m_subtype):

data['text_to_relocate'].append(rel_path_name)

added = True

if not added:

data['other'].append(path_name)

"""

manifest = get_buildfile_manifest(spec)

prefix_to_hash = dict()

prefix_to_hash[str(spec.package.prefix)] = spec.dag_hash()

deps = spack.build_environment.get_rpath_deps(spec.package)

for d in deps:

prefix_to_hash[str(d.prefix)] = d.dag_hash()

# Create buildinfo data and write it to disk

buildinfo = {}

buildinfo['sbang_install_path'] = spack.hooks.sbang.sbang_install_path()

buildinfo['relative_rpaths'] = rel

buildinfo['buildpath'] = spack.store.layout.root

buildinfo['spackprefix'] = spack.paths.prefix

buildinfo['relative_prefix'] = os.path.relpath(

spec.prefix, spack.store.layout.root)

buildinfo['relocate_textfiles'] = manifest['text_to_relocate']

buildinfo['relocate_binaries'] = manifest['binary_to_relocate']

buildinfo['relocate_links'] = manifest['link_to_relocate']

buildinfo['prefix_to_hash'] = prefix_to_hash

filename = buildinfo_file_name(workdir)

with open(filename, 'w') as outfile:

"""

return "%s/%s/%s-%s" % (spec.architecture,

str(spec.compiler).replace("@", "-"),

"""

return "%s-%s-%s-%s-%s%s" % (spec.architecture,

str(spec.compiler).replace("@", "-"),

spec.name,

spec.version,

spec.dag_hash(),

"""

return os.path.join(tarball_directory_name(spec),

# calculate sha256 hash of tar file

block_size = 65536

hasher = hashlib.sha256()

with open(file, 'rb') as tfile:

buf = tfile.read(block_size)

while len(buf) > 0:

hasher.update(buf)

buf = tfile.read(block_size)

if key is None:

keys = spack.util.gpg.signing_keys()

if len(keys) == 1:

key = keys[0]

if len(keys) > 1:

raise PickKeyException(str(keys))

if len(keys) == 0:

raise NoKeyException(

"No default key available for signing.\n"

"Use spack gpg init and spack gpg create"

" to create a default key.")

if os.path.exists('%s.asc' % specfile_path):

if force:

os.remove('%s.asc' % specfile_path)

else:

raise NoOverwriteException('%s.asc' % specfile_path)

key = select_signing_key(key)

"""Create the build cache index page.

try:

file_list = (

entry

for entry in web_util.list_url(cache_prefix)

if entry.endswith('.yaml') or entry.endswith('spec.json'))

except KeyError as inst:

msg = 'No packages at {0}: {1}'.format(cache_prefix, inst)

tty.warn(msg)

return

except Exception as err:

# If we got some kind of S3 (access denied or other connection

# error), the first non boto-specific class in the exception

# hierarchy is Exception. Just print a warning and return

msg = 'Encountered problem listing packages at {0}: {1}'.format(

cache_prefix, err)

tty.warn(msg)

return

tty.debug('Retrieving spec descriptor files from {0} to build index'.format(

cache_prefix))

all_mirror_specs = {}

for file_path in file_list:

try:

spec_url = url_util.join(cache_prefix, file_path)

tty.debug('fetching {0}'.format(spec_url))

_, _, spec_file = web_util.read_from_url(spec_url)

spec_file_contents = codecs.getreader('utf-8')(spec_file).read()

# Need full spec.json name or this gets confused with index.json.

if spec_url.endswith('.json'):

spec_dict = sjson.load(spec_file_contents)

s = Spec.from_json(spec_file_contents)

elif spec_url.endswith('.yaml'):

spec_dict = syaml.load(spec_file_contents)

s = Spec.from_yaml(spec_file_contents)

all_mirror_specs[s.dag_hash()] = {

'spec_url': spec_url,

'spec': s,

'num_deps': len(list(s.traverse(root=False))),

'binary_cache_checksum': spec_dict['binary_cache_checksum'],

'buildinfo': spec_dict['buildinfo'],

}

except (URLError, web_util.SpackWebError) as url_err:

tty.error('Error reading specfile: {0}'.format(file_path))

tty.error(url_err)

sorted_specs = sorted(all_mirror_specs.keys(),

key=lambda k: all_mirror_specs[k]['num_deps'])

tmpdir = tempfile.mkdtemp()

db_root_dir = os.path.join(tmpdir, 'db_root')

db = spack_db.Database(None, db_dir=db_root_dir,

enable_transaction_locking=False,

record_fields=['spec', 'ref_count', 'in_buildcache'])

try:

tty.debug('Specs sorted by number of dependencies:')

for dag_hash in sorted_specs:

spec_record = all_mirror_specs[dag_hash]

s = spec_record['spec']

num_deps = spec_record['num_deps']

tty.debug(' {0}/{1} -> {2}'.format(

s.name, dag_hash[:7], num_deps))

if num_deps > 0:

# Check each of this spec's dependencies (which we have already

# processed), as they are the source of truth for their own

# full hash. If the full hash we have for any deps does not

# match what those deps have themselves, then we need to splice

# this spec with those deps, and push this spliced spec

# (spec.json file) back to the mirror, as well as update the

# all_mirror_specs dictionary with this spliced spec.

to_splice = []

for dep in s.dependencies():

dep_dag_hash = dep.dag_hash()

if dep_dag_hash in all_mirror_specs:

true_dep = all_mirror_specs[dep_dag_hash]['spec']

if true_dep.full_hash() != dep.full_hash():

to_splice.append(true_dep)

if to_splice:

tty.debug(' needs the following deps spliced:')

for true_dep in to_splice:

tty.debug(' {0}/{1}'.format(

true_dep.name, true_dep.dag_hash()[:7]))

s = s.splice(true_dep, True)

# Push this spliced spec back to the mirror

spliced_spec_dict = s.to_dict(hash=ht.full_hash)

for key in ['binary_cache_checksum', 'buildinfo']:

spliced_spec_dict[key] = spec_record[key]

temp_json_path = os.path.join(tmpdir, 'spliced.spec.json')

with open(temp_json_path, 'w') as fd:

fd.write(sjson.dump(spliced_spec_dict))

spliced_spec_url = spec_record['spec_url']

web_util.push_to_url(

temp_json_path, spliced_spec_url, keep_original=False)

tty.debug(' spliced and wrote {0}'.format(

spliced_spec_url))

spec_record['spec'] = s

db.add(s, None)

db.mark(s, 'in_buildcache', True)

# Now that we have fixed any old specfiles that might have had the wrong

# full hash for their dependencies, we can generate the index, compute

# the hash, and push those files to the mirror.

index_json_path = os.path.join(db_root_dir, 'index.json')

with open(index_json_path, 'w') as f:

db._write_to_file(f)

# Read the index back in and compute it's hash

with open(index_json_path) as f:

index_string = f.read()

index_hash = compute_hash(index_string)

# Write the hash out to a local file

index_hash_path = os.path.join(db_root_dir, 'index.json.hash')

with open(index_hash_path, 'w') as f:

f.write(index_hash)

# Push the index itself

web_util.push_to_url(

index_json_path,

url_util.join(cache_prefix, 'index.json'),

keep_original=False,

extra_args={'ContentType': 'application/json'})

# Push the hash

web_util.push_to_url(

index_hash_path,

url_util.join(cache_prefix, 'index.json.hash'),

keep_original=False,

extra_args={'ContentType': 'text/plain'})

except Exception as err:

msg = 'Encountered problem pushing package index to {0}: {1}'.format(

cache_prefix, err)

tty.warn(msg)

tty.debug('\n' + traceback.format_exc())

finally:

"""Create the key index page.

tty.debug(' '.join(('Retrieving key.pub files from',

url_util.format(key_prefix),

'to build key index')))

try:

fingerprints = (

entry[:-4]

for entry in web_util.list_url(key_prefix, recursive=False)

if entry.endswith('.pub'))

except KeyError as inst:

msg = 'No keys at {0}: {1}'.format(key_prefix, inst)

tty.warn(msg)

return

except Exception as err:

# If we got some kind of S3 (access denied or other connection

# error), the first non boto-specific class in the exception

# hierarchy is Exception. Just print a warning and return

msg = 'Encountered problem listing keys at {0}: {1}'.format(

key_prefix, err)

tty.warn(msg)

return

remove_tmpdir = False

keys_local = url_util.local_file_path(key_prefix)

if keys_local:

target = os.path.join(keys_local, 'index.json')

else:

if not tmpdir:

tmpdir = tempfile.mkdtemp()

remove_tmpdir = True

target = os.path.join(tmpdir, 'index.json')

index = {

'keys': dict(

(fingerprint, {}) for fingerprint

in sorted(set(fingerprints)))

}

with open(target, 'w') as f:

sjson.dump(index, f)

if not keys_local:

try:

web_util.push_to_url(

target,

url_util.join(key_prefix, 'index.json'),

keep_original=False,

extra_args={'ContentType': 'application/json'})

except Exception as err:

msg = 'Encountered problem pushing key index to {0}: {1}'.format(

key_prefix, err)

tty.warn(msg)

finally:

if remove_tmpdir:

spec, outdir,

force=False, relative=False, unsigned=False,

allow_root=False, key=None, regenerate_index=False

):

"""

if not spec.concrete:

raise ValueError('spec must be concrete to build tarball')

# set up some paths

tmpdir = tempfile.mkdtemp()

cache_prefix = build_cache_prefix(tmpdir)

tarfile_name = tarball_name(spec, '.tar.gz')

tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec))

tarfile_path = os.path.join(tarfile_dir, tarfile_name)