|
0
|
1 """
|
|
|
2 code for interfacing with EC2 instances:
|
|
|
3
|
|
|
4 curl http://169.254.169.254/latest/meta-data/
|
|
|
5 """
|
|
|
6
|
|
|
7 # imports
|
|
|
8 import argparse
|
|
|
9 import boto.utils
|
|
|
10 import hashlib
|
|
|
11 import hmac
|
|
|
12 import json
|
|
|
13 import os
|
|
|
14 import requests
|
|
|
15 import sys
|
|
|
16 import urllib
|
|
|
17 import urlparse
|
|
|
18 import ConfigParser
|
|
|
19 from collections import OrderedDict
|
|
|
20 from datetime import datetime
|
|
|
21
|
|
|
22
|
|
|
23 class FilesystemCredentials(object):
|
|
|
24 """
|
|
|
25 Read credentials from the filesystem. See:
|
|
|
26 - http://boto.cloudhackers.com/en/latest/boto_config_tut.html
|
|
|
27 - https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
|
|
|
28
|
|
|
29 In Unix/Linux systems, on startup, the boto library looks
|
|
|
30 for configuration files in the following locations
|
|
|
31 and in the following order:
|
|
|
32
|
|
|
33 /etc/boto.cfg - for site-wide settings that all users on this machine will use
|
|
|
34 (if profile is given) ~/.aws/credentials - for credentials shared between SDKs
|
|
|
35 (if profile is given) ~/.boto - for user-specific settings
|
|
|
36 ~/.aws/credentials - for credentials shared between SDKs
|
|
|
37 ~/.boto - for user-specific settings
|
|
|
38
|
|
|
39 """
|
|
|
40
|
|
|
41 def read_aws_credentials(self, fp, section='default'):
|
|
|
42 parser = ConfigParser.RawConfigParser()
|
|
|
43 parser.readfp(fp)
|
|
|
44 if section in parser.sections():
|
|
|
45 key = 'aws_access_key_id'
|
|
|
46 if parser.has_option(section, key):
|
|
|
47 secret = 'aws_secret_access_key'
|
|
|
48 if parser.has_option(section, secret):
|
|
|
49 return (parser.get(section, key),
|
|
|
50 parser.get(section, secret))
|
|
|
51
|
|
|
52 def __init__(self):
|
|
|
53 self.resolution = OrderedDict()
|
|
|
54 home = os.environ['HOME']
|
|
|
55 if home:
|
|
|
56 self.resolution[os.path.join(home, '.aws', 'credentials')] = self.read_aws_credentials
|
|
|
57
|
|
|
58 def __call__(self):
|
|
|
59 """
|
|
|
60 return credentials....*if* available
|
|
|
61 """
|
|
|
62
|
|
|
63 for path, method in self.resolution.items():
|
|
|
64 if os.path.isfile(path):
|
|
|
65 with open(path, 'r') as f:
|
|
|
66 credentials = method(f)
|
|
|
67 if credentials:
|
|
|
68 return credentials
|
|
|
69
|
|
|
70
|
|
|
71 class EC2Metadata(object):
|
|
|
72 """EC2 instance metadata interface"""
|
|
|
73
|
|
|
74 def __init__(self, **kwargs):
|
|
|
75 self._kwargs = kwargs
|
|
|
76
|
|
|
77 def __call__(self):
|
|
|
78 """http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html"""
|
|
|
79 return boto.utils.get_instance_metadata(**self._kwargs)
|
|
|
80
|
|
|
81 def security_credentials(self):
|
|
|
82 """
|
|
|
83 return IAM credentials for an instance, if possible
|
|
|
84
|
|
|
85 See:
|
|
|
86 http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
|
|
|
87 """
|
|
|
88 # TODO: nested dict -> object notation mapping ;
|
|
|
89 # note also this is actually a `LazyLoader` value,
|
|
|
90 # not actually a dict
|
|
|
91
|
|
|
92 return self()['iam']['security-credentials']
|
|
|
93
|
|
|
94 def credentials(self, role=None):
|
|
|
95 """return active credentials"""
|
|
|
96
|
|
|
97 security_credentials = self.security_credentials()
|
|
|
98 if not security_credentials:
|
|
|
99 raise AssertionError("No security credentials available")
|
|
|
100
|
|
|
101 roles=', '.join(sorted(security_credentials.keys()))
|
|
|
102 if role is None:
|
|
|
103 if len(security_credentials) > 1:
|
|
|
104 raise AssertionError("No role given and multiple roles found for instance: {roles}".format(roles))
|
|
|
105 role = security_credentials.keys()[0]
|
|
|
106
|
|
|
107 if role not in security_credentials:
|
|
|
108 raise KeyError("Role {role} not in available IAM roles: {roles}".format(role=role, roles=roles))
|
|
|
109
|
|
|
110 return security_credentials[role]
|
|
|
111
|
|
|
112 class AWSCredentials(FilesystemCredentials):
|
|
|
113 """
|
|
|
114 try to read credentials from the filesystem
|
|
|
115 then from ec2 metadata
|
|
|
116 """
|
|
|
117
|
|
|
118 def __call__(self):
|
|
|
119
|
|
|
120 # return filesystem crednetials, if any
|
|
|
121 credentials = FilesystemCredentials.__call__(self)
|
|
|
122 if credentials:
|
|
|
123 return credentials
|
|
|
124
|
|
|
125 # otherwise try to return credentials from metadata
|
|
|
126 metadata = EC2Metadata()
|
|
|
127 try:
|
|
|
128 ec2_credentials = metadata.credentials()
|
|
|
129 except AssertionError:
|
|
|
130 return
|
|
|
131 keys = ('AccessKeyId', 'SecretAccessKey')
|
|
|
132 if set(keys).issubset(ec2_credentials.keys()):
|
|
|
133 return [ec2_credentials[key]
|
|
|
134 for key in keys]
|
|
|
135
|
|
|
136 class SignedRequest(object):
|
|
|
137 """
|
|
|
138 Signed request using Signature Version 4
|
|
|
139
|
|
|
140 http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
|
|
141 """
|
|
|
142
|
|
|
143 # signing information:
|
|
|
144 algorithm = 'AWS4-HMAC-SHA256'
|
|
|
145 termination_string = 'aws4_request'
|
|
|
146 authorization_header = "{algorithm} Credential={access_key}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}"
|
|
|
147
|
|
|
148 # date format:
|
|
|
149 date_format = '%Y%m%d'
|
|
|
150 time_format = '%H%M%S'
|
|
|
151
|
|
|
152 ### date methods
|
|
|
153
|
|
|
154 @classmethod
|
|
|
155 def datetime_format(cls):
|
|
|
156 return '{date_format}T{time_format}Z'.format(date_format=cls.date_format,
|
|
|
157 time_format=cls.time_format)
|
|
|
158
|
|
|
159 @classmethod
|
|
|
160 def datetime(cls, _datetime=None):
|
|
|
161 """
|
|
|
162 returns formatted datetime string as appropriate
|
|
|
163 for `x-amz-date` header
|
|
|
164 """
|
|
|
165
|
|
|
166 if _datetime is None:
|
|
|
167 _datetime = datetime.utcnow()
|
|
|
168 return _datetime.strftime(cls.datetime_format())
|
|
|
169
|
|
|
170 ### constructor
|
|
|
171
|
|
|
172 def __init__(self, access_key, secret_key, region, service):
|
|
|
173 self.access_key = access_key
|
|
|
174 self.secret_key = secret_key
|
|
|
175 self.region = region
|
|
|
176 self.service = service
|
|
|
177
|
|
|
178 ### hashing methods
|
|
|
179
|
|
|
180 def hash(self, message):
|
|
|
181 """hash a `message`"""
|
|
|
182 # from e.g. http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html#sig-v4-examples-get-auth-header
|
|
|
183 return hashlib.sha256(message).hexdigest()
|
|
|
184
|
|
|
185 def sign(self, key, msg):
|
|
|
186 """
|
|
|
187 See:
|
|
|
188 http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
|
|
|
189 """
|
|
|
190 return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
|
|
|
191
|
|
|
192 def signature_key(self, date_stamp):
|
|
|
193 parts = [date_stamp,
|
|
|
194 self.region,
|
|
|
195 self.service,
|
|
|
196 self.termination_string]
|
|
|
197 signed = ('AWS4' + self.secret_key).encode('utf-8')
|
|
|
198 while parts:
|
|
|
199 signed = self.sign(signed, parts.pop(0))
|
|
|
200 return signed
|
|
|
201
|
|
|
202 ###
|
|
|
203
|
|
|
204 def credential_scope(self, date_string):
|
|
|
205 """
|
|
|
206 a string that includes the date, the region you are targeting,
|
|
|
207 the service you are requesting, and a termination string
|
|
|
208 ("aws4_request") in lowercase characters.
|
|
|
209 """
|
|
|
210
|
|
|
211 parts = [date_string,
|
|
|
212 self.region,
|
|
|
213 self.service,
|
|
|
214 self.termination_string]
|
|
|
215 # TODO: "The region and service name strings must be UTF-8 encoded."
|
|
|
216 return '/'.join(parts)
|
|
|
217
|
|
|
218 ### method for canonical components
|
|
|
219
|
|
|
220 @classmethod
|
|
|
221 def canonical_uri(cls, path):
|
|
|
222 """
|
|
|
223 The canonical URI is the URI-encoded version
|
|
|
224 of the absolute path component of the URI
|
|
|
225 """
|
|
|
226
|
|
|
227 if path == '/':
|
|
|
228 path = None
|
|
|
229 if path:
|
|
|
230 canonical_uri = urllib.quote(path, safe='')
|
|
|
231 else:
|
|
|
232 # If the absolute path is empty, use a forward slash (/)
|
|
|
233 canonical_uri = '/'
|
|
|
234 return canonical_uri
|
|
|
235
|
|
|
236 @classmethod
|
|
|
237 def canonical_query(cls, query_string):
|
|
|
238 """
|
|
|
239 returns the canonical query string
|
|
|
240 """
|
|
|
241 # TODO: currently this does not use `cls`
|
|
|
242
|
|
|
243 # split into parameter names + values
|
|
|
244 query = urlparse.parse_qs(query_string)
|
|
|
245
|
|
|
246 # make this into a more appropriate data structure for processing
|
|
|
247 keyvalues = sum([[[key, value] for value in values]
|
|
|
248 for key, values in query.items()], [])
|
|
|
249
|
|
|
250 # a. URI-encode each parameter name and value
|
|
|
251 def encode(string):
|
|
|
252 return urllib.quote(string, safe='/')
|
|
|
253 encoded = [[encode(string) for string in pair]
|
|
|
254 for pair in keyvalues]
|
|
|
255
|
|
|
256 # b. Sort the encoded parameter names by character code in ascending order (ASCII order)
|
|
|
257 encoded.sort()
|
|
|
258
|
|
|
259 # c. Build the canonical query string by starting with the first parameter name in the sorted list.
|
|
|
260 # d. For each parameter, append the URI-encoded parameter name, followed by the character '=' (ASCII code 61), followed by the URI-encoded parameter value.
|
|
|
261 # e. Append the character '&' (ASCII code 38) after each parameter value, except for the last value in the list.
|
|
|
262 retval = '&'.join(['{name}={value}'.format(name=name,
|
|
|
263 value=value)
|
|
|
264 for name, value in encoded])
|
|
|
265 return retval
|
|
|
266
|
|
|
267 @classmethod
|
|
|
268 def signed_headers(cls, headers):
|
|
|
269 """
|
|
|
270 return a list of signed headers
|
|
|
271 """
|
|
|
272 names = [name.lower() for name in headers.keys()]
|
|
|
273 names.sort()
|
|
|
274 return ';'.join(names)
|
|
|
275
|
|
|
276 @classmethod
|
|
|
277 def canonical_headers(cls, headers):
|
|
|
278 """
|
|
|
279 return canonical headers:
|
|
|
280 Construct each header according to the following rules:
|
|
|
281 * Append the lowercase header name followed by a colon.
|
|
|
282 * ...
|
|
|
283 See:
|
|
|
284 - http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
|
|
285 - http://docs.python-requests.org/en/latest/user/quickstart/#custom-headers
|
|
|
286 """
|
|
|
287
|
|
|
288 canonical_headers = []
|
|
|
289 for key, value in headers.items():
|
|
|
290
|
|
|
291 # convert header name to lowercase
|
|
|
292 key = key.lower()
|
|
|
293
|
|
|
294 # trim excess white space from the header values
|
|
|
295 value = value.strip()
|
|
|
296
|
|
|
297 # convert sequential spaces in the value to a single space.
|
|
|
298 # However, do not remove extra spaces from any values
|
|
|
299 # that are inside quotation marks.
|
|
|
300 quote = '"'
|
|
|
301 if not (value and value[0] == quote and value[-1] == quote):
|
|
|
302 value = ' '.join(value.split())
|
|
|
303
|
|
|
304 canonical_headers.append((key, value))
|
|
|
305
|
|
|
306 # check for duplicate headers
|
|
|
307 names = [name for name, value in canonical_headers]
|
|
|
308 if len(set(names)) != len(names):
|
|
|
309 raise AssertionError("You have duplicate header names :( While AWS supports this use-case, this library doesn't yet")
|
|
|
310 # Append a comma-separated list of values for that header.
|
|
|
311 # If there are duplicate headers, the values are comma-separated.
|
|
|
312 # Do not sort the values in headers that have multiple values.
|
|
|
313
|
|
|
314 # Build the canonical headers list by sorting the headers by lowercase character code
|
|
|
315 canonical_headers.sort(key=lambda x: x[0])
|
|
|
316
|
|
|
317 # return canonical headers
|
|
|
318 return canonical_headers
|
|
|
319
|
|
|
320 def __call__(self, url, method='GET', headers=None, session=None):
|
|
|
321 """create a signed request and return the response"""
|
|
|
322
|
|
|
323 if session:
|
|
|
324 raise NotImplementedError('TODO')
|
|
|
325 else:
|
|
|
326 session = requests.Session()
|
|
|
327 signed_request = self.signed_request(url,
|
|
|
328 method=method,
|
|
|
329 headers=headers)
|
|
|
330 response = session.send(signed_request)
|
|
|
331 return response
|
|
|
332
|
|
|
333 def canonical_request(self, url, headers, payload='', method='GET'):
|
|
|
334 """
|
|
|
335 Return canonical request
|
|
|
336
|
|
|
337 url: "http://k0s.org/home/documents and settings"
|
|
|
338 GET
|
|
|
339 %2Fhome%2Fdocuments%20and%20settings
|
|
|
340 ...
|
|
|
341 """
|
|
|
342
|
|
|
343 # parse the url
|
|
|
344 parsed = urlparse.urlsplit(url)
|
|
|
345
|
|
|
346 # get canonical URI
|
|
|
347 canonical_uri = self.canonical_uri(parsed.path)
|
|
|
348
|
|
|
349 # construct the canonical query string
|
|
|
350 canonical_query = self.canonical_query(parsed.query)
|
|
|
351
|
|
|
352 # get the canonical headers
|
|
|
353 canonical_headers = self.canonical_headers(headers)
|
|
|
354
|
|
|
355 # format the canonical headers
|
|
|
356 canonical_header_string = '\n'.join(['{0}:{1}'.format(*header)
|
|
|
357 for header in canonical_headers]) + '\n'
|
|
|
358
|
|
|
359 # get the signed headers
|
|
|
360 signed_headers = self.signed_headers(headers)
|
|
|
361
|
|
|
362 # get the hashed payload
|
|
|
363 hashed_payload = self.hash(payload)
|
|
|
364
|
|
|
365 # join the parts to make the request:
|
|
|
366 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
|
|
367 # CanonicalRequest =
|
|
|
368 # HTTPRequestMethod + '\n' +
|
|
|
369 # CanonicalURI + '\n' +
|
|
|
370 # CanonicalQueryString + '\n' +
|
|
|
371 # CanonicalHeaders + '\n' +
|
|
|
372 # SignedHeaders + '\n' +
|
|
|
373 # HexEncode(Hash(RequestPayload))
|
|
|
374 parts = [method,
|
|
|
375 canonical_uri,
|
|
|
376 canonical_query,
|
|
|
377 canonical_header_string,
|
|
|
378 signed_headers,
|
|
|
379 hashed_payload]
|
|
|
380 canonical_request = '\n'.join(parts)
|
|
|
381 return canonical_request
|
|
|
382
|
|
|
383 def signed_request(self, url, method='GET', headers=None):
|
|
|
384 """
|
|
|
385 prepare a request:
|
|
|
386 http://docs.python-requests.org/en/latest/user/advanced/#prepared-requests
|
|
|
387 """
|
|
|
388
|
|
|
389 # parse the URL, since we like doing that so much
|
|
|
390 parsed = urlparse.urlsplit(url)
|
|
|
391
|
|
|
392 # setup the headers
|
|
|
393 if headers is None:
|
|
|
394 headers = {}
|
|
|
395 headers = OrderedDict(headers).copy()
|
|
|
396 mapping = dict([(key.lower(), key) for key in headers])
|
|
|
397 # XXX this is..."fun"
|
|
|
398 # maybe we should just x-form everything to lowercase now?
|
|
|
399 # ensure host header is set
|
|
|
400 if 'host' not in mapping:
|
|
|
401 headers['Host'] = parsed.netloc
|
|
|
402 # add the `x-amz-date` in terms of now:
|
|
|
403 # http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html
|
|
|
404 if 'x-amz-date' not in mapping:
|
|
|
405 headers['x-amz-date'] = self.datetime()
|
|
|
406
|
|
|
407 # create a PreparedRequest
|
|
|
408 req = requests.Request(method, url, headers)
|
|
|
409 prepped = req.prepare()
|
|
|
410
|
|
|
411 # return a signed version
|
|
|
412 return self.sign_request(prepped)
|
|
|
413
|
|
|
414 def sign_request(self, request):
|
|
|
415 """
|
|
|
416 sign a request;
|
|
|
417 http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
|
|
|
418 """
|
|
|
419
|
|
|
420 # ensure that we have a `x-amz-date` header in the request
|
|
|
421 request_date = request.headers.get('x-amz-date')
|
|
|
422 # Food for thought: perhaps here is a more appropriate place
|
|
|
423 # to add the headers? probably. Likely.
|
|
|
424 if request_date is None:
|
|
|
425 raise NotImplementedError('TODO')
|
|
|
426
|
|
|
427 # get the canonical request
|
|
|
428 canonical_request = self.canonical_request(method=request.method,
|
|
|
429 url=request.url,
|
|
|
430 headers=request.headers)
|
|
|
431
|
|
|
432 # Create a digest (hash) of the canonical request
|
|
|
433 # with the same algorithm that you used to hash the payload.
|
|
|
434 hashed_request = self.hash(canonical_request)
|
|
|
435
|
|
|
436 # Create the string to sign:
|
|
|
437 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
|
|
|
438 # 1. Start with the algorithm designation
|
|
|
439 parts = [self.algorithm]
|
|
|
440 # 2. Append the request date value
|
|
|
441 parts.append(request_date) # XXX we could validate the format
|
|
|
442 # 3. Append the credential scope value
|
|
|
443 date_string = request_date.split('T')[0] # XXX could do better
|
|
|
444 credential_scope = self.credential_scope(date_string)
|
|
|
445 parts.append(credential_scope)
|
|
|
446 # 4. Append the hash of the canonical request
|
|
|
447 parts.append(hashed_request)
|
|
|
448 string_to_sign = '\n'.join(parts)
|
|
|
449
|
|
|
450 # Calculate the AWS Signature Version 4
|
|
|
451 # http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
|
|
452 # 1. Derive your signing key.
|
|
|
453 signing_key = self.signature_key(date_string)
|
|
|
454 # 2. Calculate the signature
|
|
|
455 signature = hmac.new(signing_key,
|
|
|
456 string_to_sign.encode('utf-8'),
|
|
|
457 hashlib.sha256).hexdigest()
|
|
|
458
|
|
|
459 # Add the signing information to the request
|
|
|
460 # http://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html
|
|
|
461 authorization = self.authorization_header.format(algorithm=self.algorithm,
|
|
|
462 access_key=self.access_key,
|
|
|
463 credential_scope=credential_scope,
|
|
|
464 signed_headers=self.signed_headers(request.headers),
|
|
|
465 signature=signature)
|
|
|
466 request.headers['Authorization'] = authorization
|
|
|
467
|
|
|
468 # return the prepared requests
|
|
|
469 return request
|
|
|
470
|
|
|
471
|
|
|
472 def main(args=sys.argv[1:]):
|
|
|
473 """CLI"""
|
|
|
474
|
|
|
475 # parse command line
|
|
|
476 parser = argparse.ArgumentParser(description=__doc__)
|
|
|
477 parser.add_argument('--credentials', '--print-credentials',
|
|
|
478 dest='print_credentials',
|
|
|
479 action='store_true', default=False,
|
|
|
480 help="print default credentials for instance")
|
|
|
481 parser.add_argument('--url', dest='url',
|
|
|
482 help="hit this URL with a signed HTTP GET request")
|
|
|
483 parser.add_argument('--service', dest='service', default='es',
|
|
|
484 help="AWS service to use")
|
|
|
485 parser.add_argument('--region', dest='region', default='us-west-1',
|
|
|
486 help="AWS region")
|
|
|
487 # TODO: `service` and `region` come from
|
|
|
488 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
|
|
|
489 # We need to be able to derive the region from the environment.
|
|
|
490 # It would be very nice to derive the service from, say, the host
|
|
|
491 options = parser.parse_args(args)
|
|
|
492
|
|
|
493 if options.url:
|
|
|
494
|
|
|
495 # get credentials
|
|
|
496 credentials = AWSCredentials()()
|
|
|
497 if not credentials:
|
|
|
498 parser.error("No AWS credentials found")
|
|
|
499 aws_key, aws_secret = credentials
|
|
|
500
|
|
|
501 # make a signed request to the URL and exit
|
|
|
502 request = SignedRequest(aws_key,
|
|
|
503 aws_secret,
|
|
|
504 region=options.region,
|
|
|
505 service=options.service)
|
|
|
506 response = request(options.url, method='GET')
|
|
|
507 print ('-'*10)
|
|
|
508 print (response.text)
|
|
|
509 response.raise_for_status()
|
|
|
510 return
|
|
|
511
|
|
|
512 # metadata interface
|
|
|
513 metadata = EC2Metadata()
|
|
|
514
|
|
|
515 # get desired data
|
|
|
516 if options.print_credentials:
|
|
|
517 data = metadata.credentials()
|
|
|
518 else:
|
|
|
519 data = metadata()
|
|
|
520
|
|
|
521 # display data
|
|
|
522 print (json.dumps(data, indent=2, sort_keys=True))
|
|
|
523
|
|
|
524
|
|
|
525 if __name__ == '__main__':
|
|
|
526 main()
|
