0
+ − 1 """
+ − 2 code for interfacing with EC2 instances:
+ − 3
+ − 4 curl http://169.254.169.254/latest/meta-data/
+ − 5 """
+ − 6
+ − 7 # imports
+ − 8 import argparse
+ − 9 import boto.utils
+ − 10 import hashlib
+ − 11 import hmac
+ − 12 import json
+ − 13 import os
+ − 14 import requests
+ − 15 import sys
+ − 16 import urllib
+ − 17 import urlparse
+ − 18 import ConfigParser
+ − 19 from collections import OrderedDict
+ − 20 from datetime import datetime
+ − 21
+ − 22
+ − 23 class FilesystemCredentials(object):
+ − 24 """
+ − 25 Read credentials from the filesystem. See:
+ − 26 - http://boto.cloudhackers.com/en/latest/boto_config_tut.html
+ − 27 - https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
+ − 28
+ − 29 In Unix/Linux systems, on startup, the boto library looks
+ − 30 for configuration files in the following locations
+ − 31 and in the following order:
+ − 32
+ − 33 /etc/boto.cfg - for site-wide settings that all users on this machine will use
+ − 34 (if profile is given) ~/.aws/credentials - for credentials shared between SDKs
+ − 35 (if profile is given) ~/.boto - for user-specific settings
+ − 36 ~/.aws/credentials - for credentials shared between SDKs
+ − 37 ~/.boto - for user-specific settings
+ − 38
+ − 39 """
+ − 40
+ − 41 def read_aws_credentials(self, fp, section='default'):
+ − 42 parser = ConfigParser.RawConfigParser()
+ − 43 parser.readfp(fp)
+ − 44 if section in parser.sections():
+ − 45 key = 'aws_access_key_id'
+ − 46 if parser.has_option(section, key):
+ − 47 secret = 'aws_secret_access_key'
+ − 48 if parser.has_option(section, secret):
+ − 49 return (parser.get(section, key),
+ − 50 parser.get(section, secret))
+ − 51
+ − 52 def __init__(self):
+ − 53 self.resolution = OrderedDict()
+ − 54 home = os.environ['HOME']
+ − 55 if home:
+ − 56 self.resolution[os.path.join(home, '.aws', 'credentials')] = self.read_aws_credentials
+ − 57
+ − 58 def __call__(self):
+ − 59 """
+ − 60 return credentials....*if* available
+ − 61 """
+ − 62
+ − 63 for path, method in self.resolution.items():
+ − 64 if os.path.isfile(path):
+ − 65 with open(path, 'r') as f:
+ − 66 credentials = method(f)
+ − 67 if credentials:
+ − 68 return credentials
+ − 69
+ − 70
+ − 71 class EC2Metadata(object):
+ − 72 """EC2 instance metadata interface"""
+ − 73
+ − 74 def __init__(self, **kwargs):
+ − 75 self._kwargs = kwargs
+ − 76
+ − 77 def __call__(self):
+ − 78 """http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html"""
+ − 79 return boto.utils.get_instance_metadata(**self._kwargs)
+ − 80
+ − 81 def security_credentials(self):
+ − 82 """
+ − 83 return IAM credentials for an instance, if possible
+ − 84
+ − 85 See:
+ − 86 http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
+ − 87 """
+ − 88 # TODO: nested dict -> object notation mapping ;
+ − 89 # note also this is actually a `LazyLoader` value,
+ − 90 # not actually a dict
+ − 91
+ − 92 return self()['iam']['security-credentials']
+ − 93
+ − 94 def credentials(self, role=None):
+ − 95 """return active credentials"""
+ − 96
+ − 97 security_credentials = self.security_credentials()
+ − 98 if not security_credentials:
+ − 99 raise AssertionError("No security credentials available")
+ − 100
+ − 101 roles=', '.join(sorted(security_credentials.keys()))
+ − 102 if role is None:
+ − 103 if len(security_credentials) > 1:
+ − 104 raise AssertionError("No role given and multiple roles found for instance: {roles}".format(roles))
+ − 105 role = security_credentials.keys()[0]
+ − 106
+ − 107 if role not in security_credentials:
+ − 108 raise KeyError("Role {role} not in available IAM roles: {roles}".format(role=role, roles=roles))
+ − 109
+ − 110 return security_credentials[role]
+ − 111
+ − 112 class AWSCredentials(FilesystemCredentials):
+ − 113 """
+ − 114 try to read credentials from the filesystem
+ − 115 then from ec2 metadata
+ − 116 """
+ − 117
+ − 118 def __call__(self):
+ − 119
+ − 120 # return filesystem crednetials, if any
+ − 121 credentials = FilesystemCredentials.__call__(self)
+ − 122 if credentials:
+ − 123 return credentials
+ − 124
+ − 125 # otherwise try to return credentials from metadata
+ − 126 metadata = EC2Metadata()
+ − 127 try:
+ − 128 ec2_credentials = metadata.credentials()
+ − 129 except AssertionError:
+ − 130 return
+ − 131 keys = ('AccessKeyId', 'SecretAccessKey')
+ − 132 if set(keys).issubset(ec2_credentials.keys()):
+ − 133 return [ec2_credentials[key]
+ − 134 for key in keys]
+ − 135
+ − 136 class SignedRequest(object):
+ − 137 """
+ − 138 Signed request using Signature Version 4
+ − 139
+ − 140 http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+ − 141 """
+ − 142
+ − 143 # signing information:
+ − 144 algorithm = 'AWS4-HMAC-SHA256'
+ − 145 termination_string = 'aws4_request'
+ − 146 authorization_header = "{algorithm} Credential={access_key}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}"
+ − 147
+ − 148 # date format:
+ − 149 date_format = '%Y%m%d'
+ − 150 time_format = '%H%M%S'
+ − 151
+ − 152 ### date methods
+ − 153
+ − 154 @classmethod
+ − 155 def datetime_format(cls):
+ − 156 return '{date_format}T{time_format}Z'.format(date_format=cls.date_format,
+ − 157 time_format=cls.time_format)
+ − 158
+ − 159 @classmethod
+ − 160 def datetime(cls, _datetime=None):
+ − 161 """
+ − 162 returns formatted datetime string as appropriate
+ − 163 for `x-amz-date` header
+ − 164 """
+ − 165
+ − 166 if _datetime is None:
+ − 167 _datetime = datetime.utcnow()
+ − 168 return _datetime.strftime(cls.datetime_format())
+ − 169
+ − 170 ### constructor
+ − 171
+ − 172 def __init__(self, access_key, secret_key, region, service):
+ − 173 self.access_key = access_key
+ − 174 self.secret_key = secret_key
+ − 175 self.region = region
+ − 176 self.service = service
+ − 177
+ − 178 ### hashing methods
+ − 179
+ − 180 def hash(self, message):
+ − 181 """hash a `message`"""
+ − 182 # from e.g. http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html#sig-v4-examples-get-auth-header
+ − 183 return hashlib.sha256(message).hexdigest()
+ − 184
+ − 185 def sign(self, key, msg):
+ − 186 """
+ − 187 See:
+ − 188 http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+ − 189 """
+ − 190 return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
+ − 191
+ − 192 def signature_key(self, date_stamp):
+ − 193 parts = [date_stamp,
+ − 194 self.region,
+ − 195 self.service,
+ − 196 self.termination_string]
+ − 197 signed = ('AWS4' + self.secret_key).encode('utf-8')
+ − 198 while parts:
+ − 199 signed = self.sign(signed, parts.pop(0))
+ − 200 return signed
+ − 201
+ − 202 ###
+ − 203
+ − 204 def credential_scope(self, date_string):
+ − 205 """
+ − 206 a string that includes the date, the region you are targeting,
+ − 207 the service you are requesting, and a termination string
+ − 208 ("aws4_request") in lowercase characters.
+ − 209 """
+ − 210
+ − 211 parts = [date_string,
+ − 212 self.region,
+ − 213 self.service,
+ − 214 self.termination_string]
+ − 215 # TODO: "The region and service name strings must be UTF-8 encoded."
+ − 216 return '/'.join(parts)
+ − 217
+ − 218 ### method for canonical components
+ − 219
+ − 220 @classmethod
+ − 221 def canonical_uri(cls, path):
+ − 222 """
+ − 223 The canonical URI is the URI-encoded version
+ − 224 of the absolute path component of the URI
+ − 225 """
+ − 226
+ − 227 if path == '/':
+ − 228 path = None
+ − 229 if path:
+ − 230 canonical_uri = urllib.quote(path, safe='')
+ − 231 else:
+ − 232 # If the absolute path is empty, use a forward slash (/)
+ − 233 canonical_uri = '/'
+ − 234 return canonical_uri
+ − 235
+ − 236 @classmethod
+ − 237 def canonical_query(cls, query_string):
+ − 238 """
+ − 239 returns the canonical query string
+ − 240 """
+ − 241 # TODO: currently this does not use `cls`
+ − 242
+ − 243 # split into parameter names + values
+ − 244 query = urlparse.parse_qs(query_string)
+ − 245
+ − 246 # make this into a more appropriate data structure for processing
+ − 247 keyvalues = sum([[[key, value] for value in values]
+ − 248 for key, values in query.items()], [])
+ − 249
+ − 250 # a. URI-encode each parameter name and value
+ − 251 def encode(string):
+ − 252 return urllib.quote(string, safe='/')
+ − 253 encoded = [[encode(string) for string in pair]
+ − 254 for pair in keyvalues]
+ − 255
+ − 256 # b. Sort the encoded parameter names by character code in ascending order (ASCII order)
+ − 257 encoded.sort()
+ − 258
+ − 259 # c. Build the canonical query string by starting with the first parameter name in the sorted list.
+ − 260 # d. For each parameter, append the URI-encoded parameter name, followed by the character '=' (ASCII code 61), followed by the URI-encoded parameter value.
+ − 261 # e. Append the character '&' (ASCII code 38) after each parameter value, except for the last value in the list.
+ − 262 retval = '&'.join(['{name}={value}'.format(name=name,
+ − 263 value=value)
+ − 264 for name, value in encoded])
+ − 265 return retval
+ − 266
+ − 267 @classmethod
+ − 268 def signed_headers(cls, headers):
+ − 269 """
+ − 270 return a list of signed headers
+ − 271 """
+ − 272 names = [name.lower() for name in headers.keys()]
+ − 273 names.sort()
+ − 274 return ';'.join(names)
+ − 275
+ − 276 @classmethod
+ − 277 def canonical_headers(cls, headers):
+ − 278 """
+ − 279 return canonical headers:
+ − 280 Construct each header according to the following rules:
+ − 281 * Append the lowercase header name followed by a colon.
+ − 282 * ...
+ − 283 See:
+ − 284 - http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+ − 285 - http://docs.python-requests.org/en/latest/user/quickstart/#custom-headers
+ − 286 """
+ − 287
+ − 288 canonical_headers = []
+ − 289 for key, value in headers.items():
+ − 290
+ − 291 # convert header name to lowercase
+ − 292 key = key.lower()
+ − 293
+ − 294 # trim excess white space from the header values
+ − 295 value = value.strip()
+ − 296
+ − 297 # convert sequential spaces in the value to a single space.
+ − 298 # However, do not remove extra spaces from any values
+ − 299 # that are inside quotation marks.
+ − 300 quote = '"'
+ − 301 if not (value and value[0] == quote and value[-1] == quote):
+ − 302 value = ' '.join(value.split())
+ − 303
+ − 304 canonical_headers.append((key, value))
+ − 305
+ − 306 # check for duplicate headers
+ − 307 names = [name for name, value in canonical_headers]
+ − 308 if len(set(names)) != len(names):
+ − 309 raise AssertionError("You have duplicate header names :( While AWS supports this use-case, this library doesn't yet")
+ − 310 # Append a comma-separated list of values for that header.
+ − 311 # If there are duplicate headers, the values are comma-separated.
+ − 312 # Do not sort the values in headers that have multiple values.
+ − 313
+ − 314 # Build the canonical headers list by sorting the headers by lowercase character code
+ − 315 canonical_headers.sort(key=lambda x: x[0])
+ − 316
+ − 317 # return canonical headers
+ − 318 return canonical_headers
+ − 319
+ − 320 def __call__(self, url, method='GET', headers=None, session=None):
+ − 321 """create a signed request and return the response"""
+ − 322
+ − 323 if session:
+ − 324 raise NotImplementedError('TODO')
+ − 325 else:
+ − 326 session = requests.Session()
+ − 327 signed_request = self.signed_request(url,
+ − 328 method=method,
+ − 329 headers=headers)
+ − 330 response = session.send(signed_request)
+ − 331 return response
+ − 332
+ − 333 def canonical_request(self, url, headers, payload='', method='GET'):
+ − 334 """
+ − 335 Return canonical request
+ − 336
+ − 337 url: "http://k0s.org/home/documents and settings"
+ − 338 GET
+ − 339 %2Fhome%2Fdocuments%20and%20settings
+ − 340 ...
+ − 341 """
+ − 342
+ − 343 # parse the url
+ − 344 parsed = urlparse.urlsplit(url)
+ − 345
+ − 346 # get canonical URI
+ − 347 canonical_uri = self.canonical_uri(parsed.path)
+ − 348
+ − 349 # construct the canonical query string
+ − 350 canonical_query = self.canonical_query(parsed.query)
+ − 351
+ − 352 # get the canonical headers
+ − 353 canonical_headers = self.canonical_headers(headers)
+ − 354
+ − 355 # format the canonical headers
+ − 356 canonical_header_string = '\n'.join(['{0}:{1}'.format(*header)
+ − 357 for header in canonical_headers]) + '\n'
+ − 358
+ − 359 # get the signed headers
+ − 360 signed_headers = self.signed_headers(headers)
+ − 361
+ − 362 # get the hashed payload
+ − 363 hashed_payload = self.hash(payload)
+ − 364
+ − 365 # join the parts to make the request:
+ − 366 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+ − 367 # CanonicalRequest =
+ − 368 # HTTPRequestMethod + '\n' +
+ − 369 # CanonicalURI + '\n' +
+ − 370 # CanonicalQueryString + '\n' +
+ − 371 # CanonicalHeaders + '\n' +
+ − 372 # SignedHeaders + '\n' +
+ − 373 # HexEncode(Hash(RequestPayload))
+ − 374 parts = [method,
+ − 375 canonical_uri,
+ − 376 canonical_query,
+ − 377 canonical_header_string,
+ − 378 signed_headers,
+ − 379 hashed_payload]
+ − 380 canonical_request = '\n'.join(parts)
+ − 381 return canonical_request
+ − 382
+ − 383 def signed_request(self, url, method='GET', headers=None):
+ − 384 """
+ − 385 prepare a request:
+ − 386 http://docs.python-requests.org/en/latest/user/advanced/#prepared-requests
+ − 387 """
+ − 388
+ − 389 # parse the URL, since we like doing that so much
+ − 390 parsed = urlparse.urlsplit(url)
+ − 391
+ − 392 # setup the headers
+ − 393 if headers is None:
+ − 394 headers = {}
+ − 395 headers = OrderedDict(headers).copy()
+ − 396 mapping = dict([(key.lower(), key) for key in headers])
+ − 397 # XXX this is..."fun"
+ − 398 # maybe we should just x-form everything to lowercase now?
+ − 399 # ensure host header is set
+ − 400 if 'host' not in mapping:
+ − 401 headers['Host'] = parsed.netloc
+ − 402 # add the `x-amz-date` in terms of now:
+ − 403 # http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html
+ − 404 if 'x-amz-date' not in mapping:
+ − 405 headers['x-amz-date'] = self.datetime()
+ − 406
+ − 407 # create a PreparedRequest
+ − 408 req = requests.Request(method, url, headers)
+ − 409 prepped = req.prepare()
+ − 410
+ − 411 # return a signed version
+ − 412 return self.sign_request(prepped)
+ − 413
+ − 414 def sign_request(self, request):
+ − 415 """
+ − 416 sign a request;
+ − 417 http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+ − 418 """
+ − 419
+ − 420 # ensure that we have a `x-amz-date` header in the request
+ − 421 request_date = request.headers.get('x-amz-date')
+ − 422 # Food for thought: perhaps here is a more appropriate place
+ − 423 # to add the headers? probably. Likely.
+ − 424 if request_date is None:
+ − 425 raise NotImplementedError('TODO')
+ − 426
+ − 427 # get the canonical request
+ − 428 canonical_request = self.canonical_request(method=request.method,
+ − 429 url=request.url,
+ − 430 headers=request.headers)
+ − 431
+ − 432 # Create a digest (hash) of the canonical request
+ − 433 # with the same algorithm that you used to hash the payload.
+ − 434 hashed_request = self.hash(canonical_request)
+ − 435
+ − 436 # Create the string to sign:
+ − 437 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+ − 438 # 1. Start with the algorithm designation
+ − 439 parts = [self.algorithm]
+ − 440 # 2. Append the request date value
+ − 441 parts.append(request_date) # XXX we could validate the format
+ − 442 # 3. Append the credential scope value
+ − 443 date_string = request_date.split('T')[0] # XXX could do better
+ − 444 credential_scope = self.credential_scope(date_string)
+ − 445 parts.append(credential_scope)
+ − 446 # 4. Append the hash of the canonical request
+ − 447 parts.append(hashed_request)
+ − 448 string_to_sign = '\n'.join(parts)
+ − 449
+ − 450 # Calculate the AWS Signature Version 4
+ − 451 # http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+ − 452 # 1. Derive your signing key.
+ − 453 signing_key = self.signature_key(date_string)
+ − 454 # 2. Calculate the signature
+ − 455 signature = hmac.new(signing_key,
+ − 456 string_to_sign.encode('utf-8'),
+ − 457 hashlib.sha256).hexdigest()
+ − 458
+ − 459 # Add the signing information to the request
+ − 460 # http://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html
+ − 461 authorization = self.authorization_header.format(algorithm=self.algorithm,
+ − 462 access_key=self.access_key,
+ − 463 credential_scope=credential_scope,
+ − 464 signed_headers=self.signed_headers(request.headers),
+ − 465 signature=signature)
+ − 466 request.headers['Authorization'] = authorization
+ − 467
+ − 468 # return the prepared requests
+ − 469 return request
+ − 470
+ − 471
+ − 472 def main(args=sys.argv[1:]):
+ − 473 """CLI"""
+ − 474
+ − 475 # parse command line
+ − 476 parser = argparse.ArgumentParser(description=__doc__)
+ − 477 parser.add_argument('--credentials', '--print-credentials',
+ − 478 dest='print_credentials',
+ − 479 action='store_true', default=False,
+ − 480 help="print default credentials for instance")
+ − 481 parser.add_argument('--url', dest='url',
+ − 482 help="hit this URL with a signed HTTP GET request")
+ − 483 parser.add_argument('--service', dest='service', default='es',
+ − 484 help="AWS service to use")
+ − 485 parser.add_argument('--region', dest='region', default='us-west-1',
+ − 486 help="AWS region")
+ − 487 # TODO: `service` and `region` come from
+ − 488 # http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+ − 489 # We need to be able to derive the region from the environment.
+ − 490 # It would be very nice to derive the service from, say, the host
+ − 491 options = parser.parse_args(args)
+ − 492
+ − 493 if options.url:
+ − 494
+ − 495 # get credentials
+ − 496 credentials = AWSCredentials()()
+ − 497 if not credentials:
+ − 498 parser.error("No AWS credentials found")
+ − 499 aws_key, aws_secret = credentials
+ − 500
+ − 501 # make a signed request to the URL and exit
+ − 502 request = SignedRequest(aws_key,
+ − 503 aws_secret,
+ − 504 region=options.region,
+ − 505 service=options.service)
+ − 506 response = request(options.url, method='GET')
+ − 507 print ('-'*10)
+ − 508 print (response.text)
+ − 509 response.raise_for_status()
+ − 510 return
+ − 511
+ − 512 # metadata interface
+ − 513 metadata = EC2Metadata()
+ − 514
+ − 515 # get desired data
+ − 516 if options.print_credentials:
+ − 517 data = metadata.credentials()
+ − 518 else:
+ − 519 data = metadata()
+ − 520
+ − 521 # display data
+ − 522 print (json.dumps(data, indent=2, sort_keys=True))
+ − 523
+ − 524
+ − 525 if __name__ == '__main__':
+ − 526 main()