forked from canonical/cloud-init
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathDataSourceEc2.py
More file actions
1228 lines (1074 loc) · 42.6 KB
/
Copy pathDataSourceEc2.py
File metadata and controls
1228 lines (1074 loc) · 42.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright (C) 2009-2010 Canonical Ltd.
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
# Copyright (C) 2012 Yahoo! Inc.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Juerg Hafliger <juerg.haefliger@hp.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This file is part of cloud-init. See LICENSE file for license information.
import copy
import logging
import os
import time
import uuid
from contextlib import suppress
from typing import Dict, List
from cloudinit import dmi, net, sources
from cloudinit import url_helper as uhelp
from cloudinit import util, warnings
from cloudinit.distros import Distro
from cloudinit.event import EventScope, EventType
from cloudinit.net import netplan
from cloudinit.net.dhcp import NoDHCPLeaseError
from cloudinit.net.ephemeral import EphemeralIPNetwork
from cloudinit.sources import NicOrder
from cloudinit.sources.helpers import ec2
LOG = logging.getLogger(__name__)
STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
STRICT_ID_DEFAULT = "warn"
class CloudNames:
ALIYUN = "aliyun"
AWS = "aws"
BRIGHTBOX = "brightbox"
ZSTACK = "zstack"
E24CLOUD = "e24cloud"
OUTSCALE = "outscale"
# UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false',
# then an attempt at the Ec2 Metadata service will be made.
UNKNOWN = "unknown"
# NO_EC2_METADATA indicates this platform does not have a Ec2 metadata
# service available. No attempt at the Ec2 Metadata service will be made.
NO_EC2_METADATA = "no-ec2-metadata"
# Drop when LP: #1988157 tag handling is fixed
def skip_404_tag_errors(exception):
return exception.code == 404 and "meta-data/tags/" in exception.url
# Cloud platforms that support IMDSv2 style metadata server
IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS, CloudNames.ALIYUN]
# Only trigger hook-hotplug on NICs with Ec2 drivers. Avoid triggering
# it on docker virtual NICs and the like. LP: #1946003
_EXTRA_HOTPLUG_UDEV_RULES = """
ENV{ID_NET_DRIVER}=="vif|ena|ixgbevf", GOTO="cloudinit_hook"
GOTO="cloudinit_end"
"""
class DataSourceEc2(sources.DataSource):
dsname = "Ec2"
# Default metadata urls that will be used if none are provided
# They will be checked for 'resolveability' and some of the
# following may be discarded if they do not resolve
metadata_urls = [
"http://169.254.169.254",
"http://[fd00:ec2::254]",
"http://instance-data.:8773",
]
# The minimum supported metadata_version from the ec2 metadata apis
min_metadata_version = "2009-04-04"
# Priority ordered list of additional metadata versions which will be tried
# for extended metadata content. IPv6 support comes in 2016-09-02.
# Tags support comes in 2021-03-23.
extended_metadata_versions: List[str] = [
"2021-03-23",
"2018-09-24",
"2016-09-02",
]
# Setup read_url parameters per get_url_params.
url_max_wait = 120
url_timeout = 50
_api_token = None # API token for accessing the metadata service
_network_config = sources.UNSET # Used to cache calculated network cfg v1
# Whether we want to get network configuration from the metadata service.
perform_dhcp_setup = False
supported_update_events = {
EventScope.NETWORK: {
EventType.BOOT_NEW_INSTANCE,
EventType.BOOT,
EventType.BOOT_LEGACY,
EventType.HOTPLUG,
}
}
default_update_events = {
EventScope.NETWORK: {
EventType.BOOT_NEW_INSTANCE,
EventType.HOTPLUG,
}
}
extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES
def __init__(self, sys_cfg, distro, paths):
super(DataSourceEc2, self).__init__(sys_cfg, distro, paths)
self.metadata_address = None
self.identity = None
self._fallback_nic_order = NicOrder.MAC
def _unpickle(self, ci_pkl_version: int) -> None:
super()._unpickle(ci_pkl_version)
self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES
self._fallback_nic_order = NicOrder.MAC
def _get_cloud_name(self):
"""Return the cloud name as identified during _get_data."""
return identify_platform()
def _get_data(self):
strict_mode, _sleep = read_strict_mode(
util.get_cfg_by_path(
self.sys_cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT
),
("warn", None),
)
LOG.debug(
"strict_mode: %s, cloud_name=%s cloud_platform=%s",
strict_mode,
self.cloud_name,
self.platform,
)
if strict_mode == "true" and self.cloud_name == CloudNames.UNKNOWN:
return False
elif self.cloud_name == CloudNames.NO_EC2_METADATA:
return False
if self.perform_dhcp_setup: # Setup networking in init-local stage.
if util.is_FreeBSD():
LOG.debug("FreeBSD doesn't support running dhclient with -sf")
return False
try:
with EphemeralIPNetwork(
self.distro,
self.distro.fallback_interface,
ipv4=True,
ipv6=True,
) as netw:
state_msg = f" {netw.state_msg}" if netw.state_msg else ""
self._crawled_metadata = util.log_time(
logfunc=LOG.debug,
msg=f"Crawl of metadata service{state_msg}",
func=self.crawl_metadata,
)
except NoDHCPLeaseError:
return False
else:
self._crawled_metadata = util.log_time(
logfunc=LOG.debug,
msg="Crawl of metadata service",
func=self.crawl_metadata,
)
if not self._crawled_metadata:
return False
self.metadata = self._crawled_metadata.get("meta-data", None)
self.userdata_raw = self._crawled_metadata.get("user-data", None)
self.identity = (
self._crawled_metadata.get("dynamic", {})
.get("instance-identity", {})
.get("document", {})
)
return True
def is_classic_instance(self):
"""Report if this instance type is Ec2 Classic (non-vpc)."""
if not self.metadata:
# Can return False on inconclusive as we are also called in
# network_config where metadata will be present.
# Secondary call site is in packaging postinst script.
return False
ifaces_md = self.metadata.get("network", {}).get("interfaces", {})
for _mac, mac_data in ifaces_md.get("macs", {}).items():
if "vpc-id" in mac_data:
return False
return True
@property
def launch_index(self):
if not self.metadata:
return None
return self.metadata.get("ami-launch-index")
@property
def platform(self):
if not self._platform_type:
self._platform_type = DataSourceEc2.dsname.lower()
return self._platform_type
# IMDSv2 related parameters from the ec2 metadata api document
@property
def api_token_route(self):
return "latest/api/token"
@property
def imdsv2_token_ttl_seconds(self):
return "21600"
@property
def imdsv2_token_put_header(self):
return "X-aws-ec2-metadata-token"
@property
def imdsv2_token_req_header(self):
return self.imdsv2_token_put_header + "-ttl-seconds"
@property
def imdsv2_token_redact(self):
return [self.imdsv2_token_put_header, self.imdsv2_token_req_header]
def get_metadata_api_version(self):
"""Get the best supported api version from the metadata service.
Loop through all extended support metadata versions in order and
return the most-fully featured metadata api version discovered.
If extended_metadata_versions aren't present, return the datasource's
min_metadata_version.
"""
# Assumes metadata service is already up
url_tmpl = "{0}/{1}/meta-data/instance-id"
headers = self._get_headers()
for api_ver in self.extended_metadata_versions:
url = url_tmpl.format(self.metadata_address, api_ver)
try:
resp = uhelp.readurl(
url=url,
headers=headers,
headers_redact=self.imdsv2_token_redact,
)
except uhelp.UrlError as e:
LOG.debug("url %s raised exception %s", url, e)
else:
if resp.code == 200:
LOG.debug("Found preferred metadata version %s", api_ver)
return api_ver
elif resp.code == 404:
msg = "Metadata api version %s not present. Headers: %s"
LOG.debug(msg, api_ver, resp.headers)
return self.min_metadata_version
def get_instance_id(self):
if self.cloud_name == CloudNames.AWS:
# Prefer the ID from the instance identity document, but fall back
if not getattr(self, "identity", None):
# If re-using cached datasource, it's get_data run didn't
# setup self.identity. So we need to do that now.
api_version = self.get_metadata_api_version()
self.identity = ec2.get_instance_identity(
api_version,
self.metadata_address,
headers_cb=self._get_headers,
headers_redact=self.imdsv2_token_redact,
exception_cb=self._refresh_stale_aws_token_cb,
).get("document", {})
return self.identity.get(
"instanceId", self.metadata["instance-id"]
)
else:
return self.metadata["instance-id"]
def _maybe_fetch_api_token(self, mdurls):
"""Get an API token for EC2 Instance Metadata Service.
On EC2. IMDS will always answer an API token, unless
the instance owner has disabled the IMDS HTTP endpoint or
the network topology conflicts with the configured hop-limit.
"""
if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS:
return
urls = []
url2base = {}
url_path = self.api_token_route
request_method = "PUT"
for url in mdurls:
cur = "{0}/{1}".format(url, url_path)
urls.append(cur)
url2base[cur] = url
# use the self._imds_exception_cb to check for Read errors
LOG.debug("Fetching Ec2 IMDSv2 API Token")
response = None
url = None
url_params = self.get_url_params()
try:
url, response = uhelp.wait_for_url(
urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warning,
headers_cb=self._get_headers,
exception_cb=self._imds_exception_cb,
request_method=request_method,
headers_redact=self.imdsv2_token_redact,
connect_synchronously=False,
)
except uhelp.UrlError:
# We use the raised exception to interrupt the retry loop.
# Nothing else to do here.
pass
if url and response:
self._api_token = response
return url2base[url]
# If we get here, then wait_for_url timed out, waiting for IMDS
# or the IMDS HTTP endpoint is disabled
return None
def wait_for_metadata_service(self):
urls = []
start_time = 0
mcfg = self.ds_cfg
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", self.metadata_urls)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug(
"Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))),
)
if len(filtered):
mdurls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
mdurls = self.metadata_urls
# try the api token path first
metadata_address = self._maybe_fetch_api_token(mdurls)
# When running on EC2, we always access IMDS with an API token.
# If we could not get an API token, then we assume the IMDS
# endpoint was disabled and we move on without a data source.
# Fallback to IMDSv1 if not running on EC2
if (
not metadata_address
and self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS
):
# if we can't get a token, use instance-id path
url2base = {}
url_path = "{ver}/meta-data/instance-id".format(
ver=self.min_metadata_version
)
request_method = "GET"
for url in mdurls:
cur = "{0}/{1}".format(url, url_path)
urls.append(cur)
url2base[cur] = url
start_time = time.monotonic()
url, _ = uhelp.wait_for_url(
urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warning,
headers_redact=self.imdsv2_token_redact,
headers_cb=self._get_headers,
request_method=request_method,
)
if url:
metadata_address = url2base[url]
if metadata_address:
self.metadata_address = metadata_address
LOG.debug("Using metadata source: '%s'", self.metadata_address)
elif self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS:
LOG.warning("IMDS's HTTP endpoint is probably disabled")
else:
LOG.critical(
"Giving up on md from %s after %s seconds",
urls,
int(time.monotonic() - start_time),
)
return bool(metadata_address)
def device_name_to_device(self, name):
# Consult metadata service, that has
# ephemeral0: sdb
# and return 'sdb' for input 'ephemeral0'
if "block-device-mapping" not in self.metadata:
return None
# Example:
# 'block-device-mapping':
# {'ami': '/dev/sda1',
# 'ephemeral0': '/dev/sdb',
# 'root': '/dev/sda1'}
found = None
bdm = self.metadata["block-device-mapping"]
if not isinstance(bdm, dict):
LOG.debug("block-device-mapping not a dictionary: '%s'", bdm)
return None
for entname, device in bdm.items():
if entname == name:
found = device
break
# LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0'
if entname == "ephemeral" and name == "ephemeral0":
found = device
if found is None:
LOG.debug("Unable to convert %s to a device", name)
return None
ofound = found
if not found.startswith("/"):
found = "/dev/%s" % found
if os.path.exists(found):
return found
remapped = self._remap_device(os.path.basename(found))
if remapped:
LOG.debug("Remapped device name %s => %s", found, remapped)
return remapped
# On t1.micro, ephemeral0 will appear in block-device-mapping from
# metadata, but it will not exist on disk (and never will)
# at this point, we've verified that the path did not exist
# in the special case of 'ephemeral0' return None to avoid bogus
# fstab entry (LP: #744019)
if name == "ephemeral0":
return None
return ofound
@property
def availability_zone(self):
try:
if self.cloud_name == CloudNames.AWS:
return self.identity.get(
"availabilityZone",
self.metadata["placement"]["availability-zone"],
)
else:
return self.metadata["placement"]["availability-zone"]
except KeyError:
return None
@property
def region(self):
if self.cloud_name == CloudNames.AWS:
region = self.identity.get("region")
# Fallback to trimming the availability zone if region is missing
if self.availability_zone and not region:
region = self.availability_zone[:-1]
return region
else:
az = self.availability_zone
if az is not None:
return az[:-1]
return None
def activate(self, cfg, is_new_instance):
if not is_new_instance:
return
if self.cloud_name == CloudNames.UNKNOWN:
warn_if_necessary(
util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
cfg,
)
@property
def network_config(self):
"""Return a network config dict for rendering ENI or netplan files."""
if self._network_config != sources.UNSET:
return self._network_config
if self.metadata is None:
# this would happen if get_data hadn't been called. leave as UNSET
LOG.warning(
"Unexpected call to network_config when metadata is None."
)
return None
result = None
no_network_metadata_on_aws = bool(
"network" not in self.metadata
and self.cloud_name == CloudNames.AWS
)
if no_network_metadata_on_aws:
LOG.debug(
"Metadata 'network' not present:"
" Refreshing stale metadata from prior to upgrade."
)
util.log_time(
logfunc=LOG.debug,
msg="Re-crawl of metadata service",
func=self.get_data,
)
iface = self.distro.fallback_interface
net_md = self.metadata.get("network")
if isinstance(net_md, dict):
# SRU_BLOCKER: xenial, bionic and eoan should default
# apply_full_imds_network_config to False to retain original
# behavior on those releases.
result = convert_ec2_metadata_network_config(
net_md,
self.distro,
fallback_nic=iface,
full_network_config=util.get_cfg_option_bool(
self.ds_cfg, "apply_full_imds_network_config", True
),
fallback_nic_order=self._fallback_nic_order,
)
# Non-VPC (aka Classic) Ec2 instances need to rewrite the
# network config file every boot due to MAC address change.
if self.is_classic_instance():
self.default_update_events = copy.deepcopy(
self.default_update_events
)
self.default_update_events[EventScope.NETWORK].add(
EventType.BOOT
)
self.default_update_events[EventScope.NETWORK].add(
EventType.BOOT_LEGACY
)
else:
LOG.warning("Metadata 'network' key not valid: %s.", net_md)
self._network_config = result
return self._network_config
def crawl_metadata(self):
"""Crawl metadata service when available.
@returns: Dictionary of crawled metadata content containing the keys:
meta-data, user-data and dynamic.
"""
if not self.wait_for_metadata_service():
return {}
api_version = self.get_metadata_api_version()
redact = self.imdsv2_token_redact
crawled_metadata = {}
if self.cloud_name in IDMSV2_SUPPORTED_CLOUD_PLATFORMS:
exc_cb = self._refresh_stale_aws_token_cb
exc_cb_ud = self._skip_or_refresh_stale_aws_token_cb
skip_cb = None
elif self.cloud_name == CloudNames.OUTSCALE:
exc_cb = exc_cb_ud = None
skip_cb = skip_404_tag_errors
else:
exc_cb = exc_cb_ud = skip_cb = None
try:
raw_userdata = ec2.get_instance_userdata(
api_version,
self.metadata_address,
headers_cb=self._get_headers,
headers_redact=redact,
exception_cb=exc_cb_ud,
)
crawled_metadata["user-data"] = util.maybe_b64decode(raw_userdata)
crawled_metadata["meta-data"] = ec2.get_instance_metadata(
api_version,
self.metadata_address,
headers_cb=self._get_headers,
headers_redact=redact,
exception_cb=exc_cb,
retrieval_exception_ignore_cb=skip_cb,
)
if self.cloud_name == CloudNames.AWS:
identity = ec2.get_instance_identity(
api_version,
self.metadata_address,
headers_cb=self._get_headers,
headers_redact=redact,
exception_cb=exc_cb,
)
crawled_metadata["dynamic"] = {"instance-identity": identity}
except Exception:
util.logexc(
LOG,
"Failed reading from metadata address %s",
self.metadata_address,
)
return {}
crawled_metadata["_metadata_api_version"] = api_version
return crawled_metadata
def _refresh_api_token(self, seconds=None):
"""Request new metadata API token.
@param seconds: The lifetime of the token in seconds
@return: The API token or None if unavailable.
"""
if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS:
return None
if seconds is None:
seconds = self.imdsv2_token_ttl_seconds
LOG.debug("Refreshing Ec2 metadata API token")
request_header = {self.imdsv2_token_req_header: seconds}
token_url = "{}/{}".format(self.metadata_address, self.api_token_route)
try:
response = uhelp.readurl(
token_url,
headers=request_header,
headers_redact=self.imdsv2_token_redact,
request_method="PUT",
)
except uhelp.UrlError as e:
LOG.warning(
"Unable to get API token: %s raised exception %s", token_url, e
)
return None
return response.contents
def _skip_or_refresh_stale_aws_token_cb(self, msg, exception):
"""Callback will not retry on SKIP_USERDATA_CODES or if no token
is available."""
retry = ec2.skip_retry_on_codes(
ec2.SKIP_USERDATA_CODES, msg, exception
)
if not retry:
return False # False raises exception
return self._refresh_stale_aws_token_cb(msg, exception)
def _refresh_stale_aws_token_cb(self, msg, exception):
"""Exception handler for Ec2 to refresh token if token is stale."""
if isinstance(exception, uhelp.UrlError) and exception.code == 401:
# With _api_token as None, _get_headers will _refresh_api_token.
LOG.debug("Clearing cached Ec2 API token due to expiry")
self._api_token = None
return True # always retry
def _imds_exception_cb(self, msg, exception=None):
"""Fail quickly on proper AWS if IMDSv2 rejects API token request
Guidance from Amazon is that if IMDSv2 had disabled token requests
by returning a 403, or cloud-init malformed requests resulting in
other 40X errors, we want the datasource detection to fail quickly
without retries as those symptoms will likely not be resolved by
retries.
Exceptions such as requests.ConnectionError due to IMDS being
temporarily unroutable or unavailable will still retry due to the
callsite wait_for_url.
"""
if isinstance(exception, uhelp.UrlError):
# requests.ConnectionError will have exception.code == None
if exception.code and exception.code >= 400:
if exception.code == 403:
LOG.warning(
"Ec2 IMDS endpoint returned a 403 error. "
"HTTP endpoint is disabled. Aborting."
)
else:
LOG.warning(
"Fatal error while requesting Ec2 IMDSv2 API tokens"
)
raise exception
def _get_headers(self, url=""):
"""Return a dict of headers for accessing a url.
If _api_token is unset on AWS, attempt to refresh the token via a PUT
and then return the updated token header.
"""
if self.cloud_name not in IDMSV2_SUPPORTED_CLOUD_PLATFORMS:
return {}
# Request a 6 hour token if URL is api_token_route
request_token_header = {
self.imdsv2_token_req_header: self.imdsv2_token_ttl_seconds
}
if self.api_token_route in url:
return request_token_header
if not self._api_token:
# If we don't yet have an API token, get one via a PUT against
# api_token_route. This _api_token may get unset by a 403 due
# to an invalid or expired token
self._api_token = self._refresh_api_token()
if not self._api_token:
return {}
return {self.imdsv2_token_put_header: self._api_token}
class DataSourceEc2Local(DataSourceEc2):
"""Datasource run at init-local which sets up network to query metadata.
In init-local, no network is available. This subclass sets up minimal
networking with dhclient on a viable nic so that it can talk to the
metadata service. If the metadata service provides network configuration
then render the network configuration for that instance based on metadata.
"""
perform_dhcp_setup = True # Use dhcp before querying metadata
def get_data(self):
supported_platforms = (CloudNames.AWS, CloudNames.OUTSCALE)
if self.cloud_name not in supported_platforms:
LOG.debug(
"Local Ec2 mode only supported on %s, not %s",
supported_platforms,
self.cloud_name,
)
return False
return super(DataSourceEc2Local, self).get_data()
def read_strict_mode(cfgval, default):
try:
return parse_strict_mode(cfgval)
except ValueError as e:
LOG.warning(e)
return default
def parse_strict_mode(cfgval):
# given a mode like:
# true, false, warn,[sleep]
# return tuple with string mode (true|false|warn) and sleep.
if cfgval is True:
return "true", None
if cfgval is False:
return "false", None
if not cfgval:
return "warn", 0
mode, _, sleep = cfgval.partition(",")
if mode not in ("true", "false", "warn"):
raise ValueError(
"Invalid mode '%s' in strict_id setting '%s': "
"Expected one of 'true', 'false', 'warn'." % (mode, cfgval)
)
if sleep:
try:
sleep = int(sleep)
except ValueError as e:
raise ValueError(
"Invalid sleep '%s' in strict_id setting '%s': not an integer"
% (sleep, cfgval)
) from e
else:
sleep = None
return mode, sleep
def warn_if_necessary(cfgval, cfg):
try:
mode, sleep = parse_strict_mode(cfgval)
except ValueError as e:
LOG.warning(e)
return
if mode == "false":
return
warnings.show_warning("non_ec2_md", cfg, mode=True, sleep=sleep)
def identify_aliyun(data):
if data["product_name"] == "Alibaba Cloud ECS":
return CloudNames.ALIYUN
def identify_aws(data):
# data is a dictionary returned by _collect_platform_data.
uuid_str = data["uuid"]
if uuid_str.startswith("ec2"):
# example same-endian uuid:
# EC2E1916-9099-7CAF-FD21-012345ABCDEF
return CloudNames.AWS
with suppress(ValueError):
if uuid.UUID(uuid_str).bytes_le.hex().startswith("ec2"):
# check for other endianness
# example other-endian uuid:
# 45E12AEC-DCD1-B213-94ED-012345ABCDEF
return CloudNames.AWS
return None
def identify_brightbox(data):
if data["serial"].endswith(".brightbox.com"):
return CloudNames.BRIGHTBOX
def identify_zstack(data):
if data["asset_tag"].endswith(".zstack.io"):
return CloudNames.ZSTACK
def identify_e24cloud(data):
if data["vendor"] == "e24cloud":
return CloudNames.E24CLOUD
def identify_outscale(data):
if (
data["product_name"] == "3DS Outscale VM".lower()
and data["vendor"] == "3DS Outscale".lower()
):
return CloudNames.OUTSCALE
def identify_platform():
# identify the platform and return an entry in CloudNames.
data = _collect_platform_data()
checks = (
identify_aws,
identify_brightbox,
identify_zstack,
identify_e24cloud,
identify_outscale,
identify_aliyun,
lambda x: CloudNames.UNKNOWN,
)
for checker in checks:
try:
result = checker(data)
if result:
return result
except Exception as e:
LOG.warning(
"calling %s with %s raised exception: %s", checker, data, e
)
def _collect_platform_data():
"""Returns a dictionary of platform info from dmi or /sys/hypervisor.
Keys in the dictionary are as follows:
uuid: system-uuid from dmi or /sys/hypervisor
serial: dmi 'system-serial-number' (/sys/.../product_serial)
asset_tag: 'dmidecode -s chassis-asset-tag'
vendor: dmi 'system-manufacturer' (/sys/.../sys_vendor)
product_name: dmi 'system-product-name' (/sys/.../system-manufacturer)
On Ec2 instances experimentation is that product_serial is upper case,
and product_uuid is lower case. This returns lower case values for both.
"""
uuid = None
with suppress(OSError, UnicodeDecodeError):
uuid = util.load_text_file("/sys/hypervisor/uuid").strip()
uuid = uuid or dmi.read_dmi_data("system-uuid") or ""
serial = dmi.read_dmi_data("system-serial-number") or ""
asset_tag = dmi.read_dmi_data("chassis-asset-tag") or ""
vendor = dmi.read_dmi_data("system-manufacturer") or ""
product_name = dmi.read_dmi_data("system-product-name") or ""
return {
"uuid": uuid.lower(),
"serial": serial.lower(),
"asset_tag": asset_tag.lower(),
"vendor": vendor.lower(),
"product_name": product_name.lower(),
}
def _build_nic_order(
macs_metadata: Dict[str, Dict],
macs_to_nics: Dict[str, str],
fallback_nic_order: NicOrder = NicOrder.MAC,
) -> Dict[str, int]:
"""
Builds a dictionary containing macs as keys and nic orders as values,
taking into account `network-card` and `device-number` if present.
Note that the first NIC will be the primary NIC as it will be the one with
[network-card] == 0 and device-number == 0 if present.
@param macs_metadata: dictionary with mac address as key and contents like:
{"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
@macs_to_nics: dictionary with mac address as key and nic name as value
@return: Dictionary with macs as keys and nic orders as values.
"""
nic_order: Dict[str, int] = {}
if len(macs_to_nics) == 0 or len(macs_metadata) == 0:
return nic_order
valid_macs_metadata = filter(
# filter out nics without metadata (not a physical nic)
lambda mmd: mmd[1] is not None,
# filter by macs
map(
lambda mac: (mac, macs_metadata.get(mac), macs_to_nics[mac]),
macs_to_nics.keys(),
),
)
def _get_key_as_int_or(dikt, key, alt_value):
value = dikt.get(key, None)
if value is not None:
return int(value)
return alt_value
# Sort by (network_card, device_index) as some instances could have
# multiple network cards with repeated device indexes.
#
# On platforms where network-card and device-number are not present,
# as AliYun, the order will be by mac, as before the introduction of this
# function.
return {
mac: i
for i, (mac, _mac_metadata, _nic_name) in enumerate(
sorted(
valid_macs_metadata,
key=lambda mmd: (
_get_key_as_int_or(
mmd[1], "network-card", float("infinity")
),
_get_key_as_int_or(
mmd[1], "device-number", float("infinity")
),
(
mmd[2]
if fallback_nic_order == NicOrder.NIC_NAME
else mmd[0]
),
),
)
)
}
def _configure_policy_routing(
dev_config: dict,
*,
nic_name: str,
nic_metadata: dict,
distro: Distro,
is_ipv4: bool,
table: int,
) -> None:
"""
Configure policy-based routing on secondary NICs / secondary IPs to
ensure outgoing packets are routed via the correct interface.
@param: dev_config: network cfg v2 to be updated inplace.
@param: nic_name: nic name. Only used if ipv4.
@param: nic_metadata: nic metadata from IMDS.
@param: distro: Instance of Distro. Only used if ipv4.
@param: is_ipv4: Boolean indicating if we are acting over ipv4 or not.
@param: table: Routing table id.
"""
if is_ipv4:
subnet_prefix_routes = nic_metadata.get("subnet-ipv4-cidr-block")
ips = nic_metadata.get("local-ipv4s")
else:
subnet_prefix_routes = nic_metadata.get("subnet-ipv6-cidr-blocks")
ips = nic_metadata.get("ipv6s")
if not (subnet_prefix_routes and ips):
LOG.debug(
"Not enough IMDS information to configure policy routing "
"for IPv%s",
"4" if is_ipv4 else "6",
)
return
if not dev_config.get("routes"):
dev_config["routes"] = []
if is_ipv4:
try:
lease = distro.dhcp_client.dhcp_discovery(nic_name, distro=distro)
gateway = lease["routers"]
except NoDHCPLeaseError as e:
LOG.warning(
"Could not perform dhcp discovery on %s to find its "
"gateway. Not adding default route via the gateway. "