Attachment 5914 Details for Bug 34819

[patch] ec2/lib.py with support for iops and ephemeral volumes

lib.py.patch (text/plain), 6.22 KB, created by Daniel Orrego on 2014-05-16 18:05 CEST

(hide)

Description:

Filename:

MIME Type:

Creator: Daniel Orrego

Created: 2014-05-16 18:05 CEST

Size: 6.22 KB

patch

obsolete

>Index: lib.py
>===================================================================
>--- lib.py	(revision 50347)
>+++ lib.py	(working copy)
>@@ -126,7 +126,7 @@
> 		# Create the logfile
> 		if self.logfile:
> 			log = open(self.logfile, 'a+')
>-			log.write('Created instance %s at %s\n' % (section, time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())))
>+			log.write('Creating instance %s at %s\n' % (section, time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())))
> 			log.close()
> 		self.logfile_fd = None
> 
>@@ -144,13 +144,13 @@
> 		self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
> 		start = now = time.time()
> 		# TODO: make the timeout configurable
>-		timeout = 1200
>+		timeout = 600
> 		while now - start < timeout:
> 			try:
> 				self._connect_vm()
> 				break
> 			except socket_error:
>-				self._log('Pending %d...'  % (timeout - now + start))
>+				self._log('Connecting %d...'  % (timeout - now + start))
> 				time.sleep(5)
> 				now = time.time()
> 			except paramiko.AuthenticationException:
>@@ -159,7 +159,7 @@
> 				now = time.time()
> 			except Exception, ex:
> 				self._log('Unknown error "%s"...'  % (ex,))
>-				self._log('Pending %d...'  % (timeout - now + start))
>+				self._log('Connecting %d...'  % (timeout - now + start))
> 				time.sleep(5)
> 				now = time.time()
> 		else:
>@@ -552,10 +552,14 @@
> 				'ec2_ami',
> 				'ec2_security_group',
> 				'ec2_instance_type',
>+				'ec2_instance_ebsOptimized',
> 				'ec2_keypair',
> 				'ec2_region',
> 				'ec2_subnet_id',
> 				'ec2_partition_size',
>+				'ec2_instance_store',
>+				'ec2_volume_type',
>+				'ec2_volume_iops',
> 				]
> 		for key in params:
> 			if not config.has_option(section, key):
>@@ -575,13 +579,35 @@
> 		Create explicit block device with given size.
> 		"""
> 		bdm = None
>+		#self._log('Mapping volumes...')
> 		if self.aws_cfg.get('ec2_partition_size'):
>+			opt_volume_type = 'standard'
>+			opt_iops = None
>+			if self.aws_cfg.get('ec2_volume_type'):
>+				#self._log('Adding IOPS volumes...')
>+				#TODO: make sure ebsOptimized is also active
>+				opt_volume_type = self.aws_cfg.get('ec2_volume_type')
>+				# IOPS - up to partition_size * 30 (See AWS docs)
>+				opt_iops = self.aws_cfg.get('ec2_volume_iops')
> 			dev_sda1 = blockdevicemapping.EBSBlockDeviceType(
>-					size = self.aws_cfg.get('ec2_partition_size'),
>-					delete_on_termination=True,
>-					)
>+				size = self.aws_cfg.get('ec2_partition_size'),
>+				volume_type = opt_volume_type,
>+				iops = opt_iops,
>+				delete_on_termination = True)	
> 			bdm = blockdevicemapping.BlockDeviceMapping()
> 			bdm['/dev/sda1'] = dev_sda1
>+		#self._log('instance stores %s' % self.aws_cfg.get('ec2_instance_store'))
>+		#assumming 2 - TODO: cycle for given-number-of-devices
>+		if self.aws_cfg.get('ec2_instance_store'):
>+			#self._log('Adding instance store volumes...')
>+			dev_xvdb = blockdevicemapping.BlockDeviceType()
>+			dev_xvdc = blockdevicemapping.BlockDeviceType()
>+			dev_xvdb.ephemeral_name = 'ephemeral0'
>+			dev_xvdc.ephemeral_name = 'ephemeral1'
>+			bdm = blockdevicemapping.BlockDeviceMapping()
>+			bdm['/dev/sdb'] = dev_xvdb
>+			bdm['/dev/sdc'] = dev_xvdc
>+		#self._log('...done. MappedDevices: %s' % bdm)
> 		return bdm
> 
> 	def _connect_vm(self):
>@@ -605,6 +631,7 @@
> 		user_data = '\n'.join(['%s=%s' % (v, os.getenv(v, '')) for v in env_vars])
> 
> 		self.ec2 = boto.connect_ec2(**aws_cfg)
>+
> 		reuse = self.aws_cfg.get('ec2_reuse')
> 		if reuse:
> 			reservation = self.ec2.get_all_instances(instance_ids=[reuse])[0]
>@@ -617,7 +644,7 @@
> 				user_data=user_data,
> 				security_group_ids=[self.aws_cfg['ec2_security_group']],
> 				instance_type=self.aws_cfg['ec2_instance_type'],
>-				instance_initiated_shutdown_behavior='terminate',  # 'save'
>+				instance_initiated_shutdown_behavior='terminate',
> 				block_device_map=self._get_blockdevicemapping()
> 				)
> 		else:
>@@ -628,29 +655,44 @@
> 				user_data=user_data,
> 				security_groups=[self.aws_cfg['ec2_security_group']],
> 				instance_type=self.aws_cfg['ec2_instance_type'],
>-				instance_initiated_shutdown_behavior='terminate',  # 'save'
>+				instance_initiated_shutdown_behavior='terminate',
> 				block_device_map=self._get_blockdevicemapping()
> 				)
>+		
>+		self.instance = reservation.instances[0]		
>+		self._log('Instance %s reserved. ->%s' % (self.instance.id, self.instance.state))
> 
>-		self.instance = reservation.instances[0]
>-
> 		self._wait_instance()
>-
>+		
> 		self.instance.add_tag('Name', 'Test-%s-%s' % (os.getenv('USER'), self.section))
> 		self.instance.add_tag('class', 'ucs-test')
> 		for var in env_vars:
> 			self.instance.add_tag(var.lower(), os.getenv(var, ''))
>+		#Need ebsOptimized if volume_type is io1
>+		#NOTE:launching an instance with image.run does not support ebsOptimized
>+		#modifying instance attribute:
>+		#self._log('EBS status: %s' % (self.instance.ebs_optimized))
>+		if self.aws_cfg.get('ec2_instance_ebsOptimized') == 'True':
>+			self._log('Changing EBS status: stopping instance first.')
>+			# stop VM (make sure it will only be stopped, not terminated)
>+			self.instance.modify_attribute('instanceInitiatedShutdownBehavior', 'stop')
>+			self.instance.stop()
>+			self._wait_instance(desiredState='stopped')
>+			self.instance.modify_attribute('ebsOptimized', True)
>+			self.instance.modify_attribute('instanceInitiatedShutdownBehavior', 'terminate')
>+			self.instance.start()
>+			self._wait_instance()
> 
>-	def _wait_instance(self, timeout=600):
>+	def _wait_instance(self, timeout=600, desiredState='running'):
> 		"""
> 		Wait until instance is created.
> 		"""
> 		start = now = time.time()
> 		while now - start < timeout:
>-			if self.instance.state == 'running':
>+			if self.instance.state == desiredState:
> 				break
>-			if self.instance.state == 'pending':
>-				self._log('Pending %d...' % (timeout - now + start))
>+			if self.instance.state != desiredState:
>+				self._log('State: %s, waiting %d...' % (self.instance.state, timeout - now + start))
> 			time.sleep(10)
> 
> 			try:
>@@ -661,14 +703,14 @@
> 					if error.error_code == 'InvalidInstanceID.NotFound':
> 						break
> 				else:
>-					self._log('Unexcpected error waiting for instance: %s', ex)
>+					self._log('Unexpected error waiting for instance: %s', ex)
> 					raise
> 			now = time.time()
> 		else:
> 			self._log('Timeout waiting for instance')
> 			raise
>+		self._log('OK ->%s' % (self.instance.state))
> 
>-
> def _print_process(msg):
> 	'''	Print s status line '''
> 	if len(msg) > 64:

Actions: View | Diff

Attachments on bug 34819: 5910 | 5914 | 5924