서승완 / kappa

Go to a project
Toggle navigation Toggle navigation pinning
Mitch Garnaat
cb905c47 1 parent 028e1597

Fixed some deployment issues. Also changed it so that every code deployment cre… 

…ates not just a new version but also a new alias based on the environment.  No longer use environment explicitly in names.
Inline Side-by-side
Showing 5 changed files with 141 additions and 128 deletions
...@@ -18,6 +18,8 @@ import time ...@@ -18,6 +18,8 @@ import time
18 import os 18 import os
19 import shutil 19 import shutil
20 20
21 +from botocore.exceptions import ClientError
22 +
21 import kappa.function 23 import kappa.function
22 import kappa.event_source 24 import kappa.event_source
23 import kappa.policy 25 import kappa.policy
...@@ -79,7 +81,7 @@ class Context(object): ...@@ -79,7 +81,7 @@ class Context(object):
79 81
80 @property 82 @property
81 def name(self): 83 def name(self):
82 - return '{}-{}'.format(self.config['name'], self.environment) 84 + return self.config.get('name', os.path.basename(os.getcwd()))
83 85
84 @property 86 @property
85 def profile(self): 87 def profile(self):
...@@ -184,11 +186,6 @@ class Context(object): ...@@ -184,11 +186,6 @@ class Context(object):
184 self.policy.deploy() 186 self.policy.deploy()
185 if self.role: 187 if self.role:
186 self.role.create() 188 self.role.create()
187 - # There is a consistency problem here.
188 - # If you don't wait for a bit, the function.create call
189 - # will fail because the policy has not been attached to the role.
190 - LOG.debug('Waiting for policy/role propogation')
191 - time.sleep(5)
192 self.function.deploy() 189 self.function.deploy()
193 190
194 def invoke(self, data): 191 def invoke(self, data):
...@@ -215,9 +212,6 @@ class Context(object): ...@@ -215,9 +212,6 @@ class Context(object):
215 def tail(self): 212 def tail(self):
216 return self.function.tail() 213 return self.function.tail()
217 214
218 - def tag(self, name, description):
219 - return self.function.tag(name, description)
220 -
221 def delete(self): 215 def delete(self):
222 for event_source in self.event_sources: 216 for event_source in self.event_sources:
223 event_source.remove(self.function) 217 event_source.remove(self.function)
......
...@@ -77,6 +77,33 @@ class Function(object): ...@@ -77,6 +77,33 @@ class Function(object):
77 def permissions(self): 77 def permissions(self):
78 return self._config.get('permissions', list()) 78 return self._config.get('permissions', list())
79 79
80 + @property
81 + def log(self):
82 + if self._log is None:
83 + log_group_name = '/aws/lambda/%s' % self.name
84 + self._log = kappa.log.Log(self._context, log_group_name)
85 + return self._log
86 +
87 + @property
88 + def code_sha_256(self):
89 + return self._get_response_configuration('CodeSha256')
90 +
91 + @property
92 + def arn(self):
93 + return self._get_response_configuration('FunctionArn')
94 +
95 + @property
96 + def repository_type(self):
97 + return self._get_response_code('RepositoryType')
98 +
99 + @property
100 + def location(self):
101 + return self._get_response_code('Location')
102 +
103 + @property
104 + def version(self):
105 + return self._get_response_configuration('Version')
106 +
80 def _get_response(self): 107 def _get_response(self):
81 if self._response is None: 108 if self._response is None:
82 try: 109 try:
...@@ -104,30 +131,10 @@ class Function(object): ...@@ -104,30 +131,10 @@ class Function(object):
104 value = response['Configuration'].get(key, default) 131 value = response['Configuration'].get(key, default)
105 return value 132 return value
106 133
107 - @property
108 - def code_sha_256(self):
109 - return self._get_response_configuration('CodeSha256')
110 -
111 - @property
112 - def arn(self):
113 - return self._get_response_configuration('FunctionArn')
114 -
115 - @property
116 - def repository_type(self):
117 - return self._get_response_code('RepositoryType')
118 -
119 - @property
120 - def location(self):
121 - return self._get_response_code('Location')
122 -
123 - @property
124 - def version(self):
125 - return self._get_response_configuration('Version')
126 -
127 - def exists(self):
128 - return self._get_response()
129 -
130 def _check_function_md5(self): 134 def _check_function_md5(self):
135 + # Zip up the source code and then compute the MD5 of that.
136 + # If the MD5 does not match the cached MD5, the function has
137 + # changed and needs to be updated so return True.
131 changed = True 138 changed = True
132 self._copy_config_file() 139 self._copy_config_file()
133 self.zip_lambda_function(self.zipfile_name, self.path) 140 self.zip_lambda_function(self.zipfile_name, self.path)
...@@ -146,6 +153,9 @@ class Function(object): ...@@ -146,6 +153,9 @@ class Function(object):
146 return changed 153 return changed
147 154
148 def _check_config_md5(self): 155 def _check_config_md5(self):
156 + # Compute the MD5 of all of the components of the configuration.
157 + # If the MD5 does not match the cached MD5, the configuration has
158 + # changed and needs to be updated so return True.
149 m = hashlib.md5() 159 m = hashlib.md5()
150 m.update(self.description) 160 m.update(self.description)
151 m.update(self.handler) 161 m.update(self.handler)
...@@ -163,16 +173,13 @@ class Function(object): ...@@ -163,16 +173,13 @@ class Function(object):
163 changed = False 173 changed = False
164 return changed 174 return changed
165 175
166 - @property 176 + def _copy_config_file(self):
167 - def log(self): 177 + config_name = '{}_config.json'.format(self._context.environment)
168 - if self._log is None: 178 + config_path = os.path.join(self.path, config_name)
169 - log_group_name = '/aws/lambda/%s' % self.name 179 + if os.path.exists(config_path):
170 - self._log = kappa.log.Log(self._context, log_group_name) 180 + dest_path = os.path.join(self.path, 'config.json')
171 - return self._log 181 + LOG.debug('copy %s to %s', config_path, dest_path)
172 - 182 + shutil.copy2(config_path, dest_path)
173 - def tail(self):
174 - LOG.info('tailing function: %s', self.name)
175 - return self.log.tail()
176 183
177 def _zip_lambda_dir(self, zipfile_name, lambda_dir): 184 def _zip_lambda_dir(self, zipfile_name, lambda_dir):
178 LOG.debug('_zip_lambda_dir: lambda_dir=%s', lambda_dir) 185 LOG.debug('_zip_lambda_dir: lambda_dir=%s', lambda_dir)
...@@ -202,6 +209,51 @@ class Function(object): ...@@ -202,6 +209,51 @@ class Function(object):
202 else: 209 else:
203 self._zip_lambda_file(zipfile_name, lambda_fn) 210 self._zip_lambda_file(zipfile_name, lambda_fn)
204 211
212 + def exists(self):
213 + return self._get_response()
214 +
215 + def tail(self):
216 + LOG.info('tailing function: %s', self.name)
217 + return self.log.tail()
218 +
219 + def list_aliases(self):
220 + LOG.info('listing aliases of %s', self.name)
221 + try:
222 + response = self._lambda_client.call(
223 + 'list_aliases',
224 + FunctionName=self.name,
225 + FunctionVersion=self.version)
226 + LOG.debug(response)
227 + except Exception:
228 + LOG.exception('Unable to list aliases')
229 + return response['Versions']
230 +
231 + def create_alias(self, name, description, version=None):
232 + # Find the current (latest) version by version number
233 + # First find the SHA256 of $LATEST
234 + if not version:
235 + versions = self.list_versions()
236 + for v in versions:
237 + if v['Version'] == '$LATEST':
238 + latest_sha256 = v['CodeSha256']
239 + break
240 + for v in versions:
241 + if v['Version'] != '$LATEST':
242 + if v['CodeSha256'] == latest_sha256:
243 + version = v['Version']
244 + break
245 + try:
246 + LOG.debug('creating alias %s=%s', name, version)
247 + response = self._lambda_client.call(
248 + 'create_alias',
249 + FunctionName=self.name,
250 + Description=description,
251 + FunctionVersion=version,
252 + Name=name)
253 + LOG.debug(response)
254 + except Exception:
255 + LOG.exception('Unable to create alias')
256 +
205 def add_permissions(self): 257 def add_permissions(self):
206 if self.permissions: 258 if self.permissions:
207 time.sleep(5) 259 time.sleep(5)
...@@ -224,41 +276,46 @@ class Function(object): ...@@ -224,41 +276,46 @@ class Function(object):
224 except Exception: 276 except Exception:
225 LOG.exception('Unable to add permission') 277 LOG.exception('Unable to add permission')
226 278
227 - def _copy_config_file(self):
228 - config_name = '{}_config.json'.format(self._context.environment)
229 - config_path = os.path.join(self.path, config_name)
230 - if os.path.exists(config_path):
231 - dest_path = os.path.join(self.path, 'config.json')
232 - LOG.debug('copy %s to %s', config_path, dest_path)
233 - shutil.copy2(config_path, dest_path)
234 -
235 def create(self): 279 def create(self):
236 LOG.info('creating function %s', self.name) 280 LOG.info('creating function %s', self.name)
237 self._check_function_md5() 281 self._check_function_md5()
238 self._check_config_md5() 282 self._check_config_md5()
239 - with open(self.zipfile_name, 'rb') as fp: 283 + # There is a consistency problem here.
240 - exec_role = self._context.exec_role_arn 284 + # Sometimes the role is not ready to be used by the function.
241 - LOG.debug('exec_role=%s', exec_role) 285 + ready = False
242 - try: 286 + while not ready:
243 - zipdata = fp.read() 287 + with open(self.zipfile_name, 'rb') as fp:
244 - response = self._lambda_client.call( 288 + exec_role = self._context.exec_role_arn
245 - 'create_function', 289 + LOG.debug('exec_role=%s', exec_role)
246 - FunctionName=self.name, 290 + try:
247 - Code={'ZipFile': zipdata}, 291 + zipdata = fp.read()
248 - Runtime=self.runtime, 292 + response = self._lambda_client.call(
249 - Role=exec_role, 293 + 'create_function',
250 - Handler=self.handler, 294 + FunctionName=self.name,
251 - Description=self.description, 295 + Code={'ZipFile': zipdata},
252 - Timeout=self.timeout, 296 + Runtime=self.runtime,
253 - MemorySize=self.memory_size, 297 + Role=exec_role,
254 - Publish=True) 298 + Handler=self.handler,
255 - LOG.debug(response) 299 + Description=self.description,
256 - except Exception: 300 + Timeout=self.timeout,
257 - LOG.exception('Unable to upload zip file') 301 + MemorySize=self.memory_size,
302 + Publish=True)
303 + LOG.debug(response)
304 + description = 'For stage {}'.format(
305 + self._context.environment)
306 + self.create_alias(self._context.environment, description)
307 + ready = True
308 + except ClientError, e:
309 + if 'InvalidParameterValueException' in str(e):
310 + LOG.debug('Role is not ready, waiting')
311 + time.sleep(2)
312 + except Exception:
313 + LOG.exception('Unable to upload zip file')
314 + ready = True
258 self.add_permissions() 315 self.add_permissions()
259 316
260 def update(self): 317 def update(self):
261 - LOG.info('updating %s', self.name) 318 + LOG.info('updating function %s', self.name)
262 if self._check_function_md5(): 319 if self._check_function_md5():
263 self._response = None 320 self._response = None
264 with open(self.zipfile_name, 'rb') as fp: 321 with open(self.zipfile_name, 'rb') as fp:
...@@ -272,6 +329,9 @@ class Function(object): ...@@ -272,6 +329,9 @@ class Function(object):
272 ZipFile=zipdata, 329 ZipFile=zipdata,
273 Publish=True) 330 Publish=True)
274 LOG.debug(response) 331 LOG.debug(response)
332 + self.create_alias(
333 + self._context.environment,
334 + 'For the {} stage'.format(self._context.environment))
275 except Exception: 335 except Exception:
276 LOG.exception('unable to update zip file') 336 LOG.exception('unable to update zip file')
277 337
...@@ -312,44 +372,6 @@ class Function(object): ...@@ -312,44 +372,6 @@ class Function(object):
312 LOG.exception('Unable to list versions') 372 LOG.exception('Unable to list versions')
313 return response['Versions'] 373 return response['Versions']
314 374
315 - def create_alias(self, name, description, version=None):
316 - # Find the current (latest) version by version number
317 - # First find the SHA256 of $LATEST
318 - if not version:
319 - versions = self.list_versions()
320 - for v in versions:
321 - if v['Version'] == '$LATEST':
322 - latest_sha256 = v['CodeSha256']
323 - break
324 - for v in versions:
325 - if v['Version'] != '$LATEST':
326 - if v['CodeSha256'] == latest_sha256:
327 - version = v['Version']
328 - break
329 - try:
330 - LOG.info('creating alias %s=%s', name, version)
331 - response = self._lambda_client.call(
332 - 'create_alias',
333 - FunctionName=self.name,
334 - Description=description,
335 - FunctionVersion=version,
336 - Name=name)
337 - LOG.debug(response)
338 - except Exception:
339 - LOG.exception('Unable to create alias')
340 -
341 - def list_aliases(self):
342 - LOG.info('listing aliases of %s', self.name)
343 - try:
344 - response = self._lambda_client.call(
345 - 'list_aliases',
346 - FunctionName=self.name,
347 - FunctionVersion=self.version)
348 - LOG.debug(response)
349 - except Exception:
350 - LOG.exception('Unable to list aliases')
351 - return response['Versions']
352 -
353 def tag(self, name, description): 375 def tag(self, name, description):
354 self.create_alias(name, description) 376 self.create_alias(name, description)
355 377
......
...@@ -120,6 +120,18 @@ class Policy(object): ...@@ -120,6 +120,18 @@ class Policy(object):
120 except Exception: 120 except Exception:
121 LOG.exception('Error creating new Policy version') 121 LOG.exception('Error creating new Policy version')
122 122
123 + def _check_md5(self, document):
124 + m = hashlib.md5()
125 + m.update(document)
126 + policy_md5 = m.hexdigest()
127 + cached_md5 = self._context.get_cache_value('policy_md5')
128 + LOG.debug('policy_md5: %s', policy_md5)
129 + LOG.debug('cached md5: %s', cached_md5)
130 + if policy_md5 != cached_md5:
131 + self._context.set_cache_value('policy_md5', policy_md5)
132 + return True
133 + return False
134 +
123 def deploy(self): 135 def deploy(self):
124 LOG.info('deploying policy %s', self.name) 136 LOG.info('deploying policy %s', self.name)
125 document = self.document() 137 document = self.document()
...@@ -128,19 +140,13 @@ class Policy(object): ...@@ -128,19 +140,13 @@ class Policy(object):
128 return 140 return
129 policy = self.exists() 141 policy = self.exists()
130 if policy: 142 if policy:
131 - m = hashlib.md5() 143 + if self._check_md5(document):
132 - m.update(document)
133 - policy_md5 = m.hexdigest()
134 - cached_md5 = self._context.get_cache_value('policy_md5')
135 - LOG.debug('policy_md5: %s', policy_md5)
136 - LOG.debug('cached md5: %s', cached_md5)
137 - if policy_md5 != cached_md5:
138 - self._context.set_cache_value('policy_md5', policy_md5)
139 self._add_policy_version() 144 self._add_policy_version()
140 else: 145 else:
141 LOG.info('policy unchanged') 146 LOG.info('policy unchanged')
142 else: 147 else:
143 # create a new policy 148 # create a new policy
149 + self._check_md5(document)
144 try: 150 try:
145 response = self._iam_client.call( 151 response = self._iam_client.call(
146 'create_policy', 152 'create_policy',
......
...@@ -73,7 +73,7 @@ class Role(object): ...@@ -73,7 +73,7 @@ class Role(object):
73 return None 73 return None
74 74
75 def create(self): 75 def create(self):
76 - LOG.debug('creating role %s', self.name) 76 + LOG.info('creating role %s', self.name)
77 role = self.exists() 77 role = self.exists()
78 if not role: 78 if not role:
79 try: 79 try:
...@@ -91,6 +91,8 @@ class Role(object): ...@@ -91,6 +91,8 @@ class Role(object):
91 LOG.debug(response) 91 LOG.debug(response)
92 except ClientError: 92 except ClientError:
93 LOG.exception('Error creating Role') 93 LOG.exception('Error creating Role')
94 + else:
95 + LOG.info('role already exists')
94 96
95 def delete(self): 97 def delete(self):
96 response = None 98 response = None
......
...@@ -157,14 +157,3 @@ def event_sources(ctx, command): ...@@ -157,14 +157,3 @@ def event_sources(ctx, command):
157 click.echo('enabling event sources') 157 click.echo('enabling event sources')
158 ctx.disable_event_sources() 158 ctx.disable_event_sources()
159 click.echo('done') 159 click.echo('done')
160 -
161 -
162 -@cli.command()
163 -@click.argument('name')
164 -@click.argument('description')
165 -@pass_ctx
166 -def tag(ctx, name, description):
167 - """Tag the current function version with a symbolic name"""
168 - click.echo('creating tag for function')
169 - ctx.tag(name, description)
170 - click.echo('done')
......