Mitch Garnaat

Getting event sources working again. Lots of other changes.

...@@ -2,4 +2,4 @@ include README.md ...@@ -2,4 +2,4 @@ include README.md
2 include LICENSE 2 include LICENSE
3 include requirements.txt 3 include requirements.txt
4 include kappa/_version 4 include kappa/_version
5 -recursive-include samples *.js *.yml *.cf *.json 5 +recursive-include samples *.js *.py *.yml *.cf *.json *.txt
......
...@@ -38,6 +38,10 @@ class AWSClient(object): ...@@ -38,6 +38,10 @@ class AWSClient(object):
38 def session(self): 38 def session(self):
39 return self._session 39 return self._session
40 40
41 + @property
42 + def region_name(self):
43 + return self.client.meta.region_name
44 +
41 def _create_client(self): 45 def _create_client(self):
42 client = self._session.client(self._service_name) 46 client = self._session.client(self._service_name)
43 return client 47 return client
......
...@@ -19,6 +19,7 @@ import os ...@@ -19,6 +19,7 @@ import os
19 import shutil 19 import shutil
20 20
21 import kappa.function 21 import kappa.function
22 +import kappa.restapi
22 import kappa.event_source 23 import kappa.event_source
23 import kappa.policy 24 import kappa.policy
24 import kappa.role 25 import kappa.role
...@@ -55,6 +56,11 @@ class Context(object): ...@@ -55,6 +56,11 @@ class Context(object):
55 self, self.config['environments'][self.environment]) 56 self, self.config['environments'][self.environment])
56 self.function = kappa.function.Function( 57 self.function = kappa.function.Function(
57 self, self.config['lambda']) 58 self, self.config['lambda'])
59 + if 'restapi' in self.config:
60 + self.restapi = kappa.restapi.RestApi(
61 + self, self.config['restapi'])
62 + else:
63 + self.restapi = None
58 self.event_sources = [] 64 self.event_sources = []
59 self._create_event_sources() 65 self._create_event_sources()
60 66
...@@ -82,7 +88,8 @@ class Context(object): ...@@ -82,7 +88,8 @@ class Context(object):
82 return self.cache.setdefault(self.environment, dict()).get(key) 88 return self.cache.setdefault(self.environment, dict()).get(key)
83 89
84 def set_cache_value(self, key, value): 90 def set_cache_value(self, key, value):
85 - self.cache.setdefault(self.environment, dict())[key] = value.encode('utf-8') 91 + self.cache.setdefault(
92 + self.environment, dict())[key] = value.encode('utf-8')
86 self._save_cache() 93 self._save_cache()
87 94
88 @property 95 @property
...@@ -149,8 +156,9 @@ class Context(object): ...@@ -149,8 +156,9 @@ class Context(object):
149 log.addHandler(ch) 156 log.addHandler(ch)
150 157
151 def _create_event_sources(self): 158 def _create_event_sources(self):
152 - if 'event_sources' in self.config['lambda']: 159 + env_cfg = self.config['environments'][self.environment]
153 - for event_source_cfg in self.config['lambda']['event_sources']: 160 + if 'event_sources' in env_cfg:
161 + for event_source_cfg in env_cfg['event_sources']:
154 _, _, svc, _ = event_source_cfg['arn'].split(':', 3) 162 _, _, svc, _ = event_source_cfg['arn'].split(':', 3)
155 if svc == 'kinesis': 163 if svc == 'kinesis':
156 self.event_sources.append( 164 self.event_sources.append(
...@@ -179,6 +187,14 @@ class Context(object): ...@@ -179,6 +187,14 @@ class Context(object):
179 for event_source in self.event_sources: 187 for event_source in self.event_sources:
180 event_source.update(self.function) 188 event_source.update(self.function)
181 189
190 + def enable_event_sources(self):
191 + for event_source in self.event_sources:
192 + event_source.enable(self.function)
193 +
194 + def disable_event_sources(self):
195 + for event_source in self.event_sources:
196 + event_source.enable(self.function)
197 +
182 def create(self): 198 def create(self):
183 if self.policy: 199 if self.policy:
184 self.policy.create() 200 self.policy.create()
...@@ -197,6 +213,8 @@ class Context(object): ...@@ -197,6 +213,8 @@ class Context(object):
197 if self.role: 213 if self.role:
198 self.role.create() 214 self.role.create()
199 self.function.deploy() 215 self.function.deploy()
216 + if self.restapi:
217 + self.restapi.deploy()
200 218
201 def invoke(self, data): 219 def invoke(self, data):
202 return self.function.invoke(data) 220 return self.function.invoke(data)
...@@ -227,6 +245,8 @@ class Context(object): ...@@ -227,6 +245,8 @@ class Context(object):
227 event_source.remove(self.function) 245 event_source.remove(self.function)
228 self.function.log.delete() 246 self.function.log.delete()
229 self.function.delete() 247 self.function.delete()
248 + if self.restapi:
249 + self.restapi.delete()
230 time.sleep(5) 250 time.sleep(5)
231 if self.role: 251 if self.role:
232 self.role.delete() 252 self.role.delete()
......
...@@ -49,7 +49,7 @@ class KinesisEventSource(EventSource): ...@@ -49,7 +49,7 @@ class KinesisEventSource(EventSource):
49 def __init__(self, context, config): 49 def __init__(self, context, config):
50 super(KinesisEventSource, self).__init__(context, config) 50 super(KinesisEventSource, self).__init__(context, config)
51 self._lambda = kappa.awsclient.create_client( 51 self._lambda = kappa.awsclient.create_client(
52 - 'kinesis', context.session) 52 + 'lambda', context.session)
53 53
54 def _get_uuid(self, function): 54 def _get_uuid(self, function):
55 uuid = None 55 uuid = None
...@@ -77,7 +77,7 @@ class KinesisEventSource(EventSource): ...@@ -77,7 +77,7 @@ class KinesisEventSource(EventSource):
77 LOG.exception('Unable to add event source') 77 LOG.exception('Unable to add event source')
78 78
79 def enable(self, function): 79 def enable(self, function):
80 - self.enabled = True 80 + self._config['enabled'] = True
81 try: 81 try:
82 response = self._lambda.call( 82 response = self._lambda.call(
83 'update_event_source_mapping', 83 'update_event_source_mapping',
...@@ -89,7 +89,7 @@ class KinesisEventSource(EventSource): ...@@ -89,7 +89,7 @@ class KinesisEventSource(EventSource):
89 LOG.exception('Unable to enable event source') 89 LOG.exception('Unable to enable event source')
90 90
91 def disable(self, function): 91 def disable(self, function):
92 - self.enabled = False 92 + self._config['enabled'] = False
93 try: 93 try:
94 response = self._lambda.call( 94 response = self._lambda.call(
95 'update_event_source_mapping', 95 'update_event_source_mapping',
......
...@@ -18,6 +18,7 @@ import zipfile ...@@ -18,6 +18,7 @@ import zipfile
18 import time 18 import time
19 import shutil 19 import shutil
20 import hashlib 20 import hashlib
21 +import uuid
21 22
22 from botocore.exceptions import ClientError 23 from botocore.exceptions import ClientError
23 24
...@@ -89,6 +90,10 @@ class Function(object): ...@@ -89,6 +90,10 @@ class Function(object):
89 return self._get_response_configuration('FunctionArn') 90 return self._get_response_configuration('FunctionArn')
90 91
91 @property 92 @property
93 + def alias_arn(self):
94 + return self.arn + ':{}'.format(self._context.environment)
95 +
96 + @property
92 def repository_type(self): 97 def repository_type(self):
93 return self._get_response_code('RepositoryType') 98 return self._get_response_code('RepositoryType')
94 99
...@@ -100,6 +105,12 @@ class Function(object): ...@@ -100,6 +105,12 @@ class Function(object):
100 def version(self): 105 def version(self):
101 return self._get_response_configuration('Version') 106 return self._get_response_configuration('Version')
102 107
108 + @property
109 + def deployment_uri(self):
110 + return 'https://{}.execute-api.{}.amazonaws.com/{}'.format(
111 + self.api_id, self._apigateway_client.region_name,
112 + self._context.environment)
113 +
103 def _get_response(self): 114 def _get_response(self):
104 if self._response is None: 115 if self._response is None:
105 try: 116 try:
...@@ -217,12 +228,11 @@ class Function(object): ...@@ -217,12 +228,11 @@ class Function(object):
217 try: 228 try:
218 response = self._lambda_client.call( 229 response = self._lambda_client.call(
219 'list_aliases', 230 'list_aliases',
220 - FunctionName=self.name, 231 + FunctionName=self.name)
221 - FunctionVersion=self.version)
222 LOG.debug(response) 232 LOG.debug(response)
223 except Exception: 233 except Exception:
224 LOG.exception('Unable to list aliases') 234 LOG.exception('Unable to list aliases')
225 - return response['Versions'] 235 + return response.get('Versions', list())
226 236
227 def find_latest_version(self): 237 def find_latest_version(self):
228 # Find the current (latest) version by version number 238 # Find the current (latest) version by version number
...@@ -271,27 +281,34 @@ class Function(object): ...@@ -271,27 +281,34 @@ class Function(object):
271 except Exception: 281 except Exception:
272 LOG.exception('Unable to update alias') 282 LOG.exception('Unable to update alias')
273 283
284 + def add_permission(self, action, principal,
285 + source_arn=None, source_account=None):
286 + try:
287 + kwargs = {
288 + 'FunctionName': self.name,
289 + 'Qualifier': self._context.environment,
290 + 'StatementId': str(uuid.uuid4()),
291 + 'Action': action,
292 + 'Principal': principal}
293 + if source_arn:
294 + kwargs['SourceArn'] = source_arn
295 + if source_account:
296 + kwargs['SourceAccount'] = source_account
297 + response = self._lambda_client.call(
298 + 'add_permission', **kwargs)
299 + LOG.debug(response)
300 + except Exception:
301 + LOG.exception('Unable to add permission')
302 +
274 def add_permissions(self): 303 def add_permissions(self):
275 if self.permissions: 304 if self.permissions:
276 time.sleep(5) 305 time.sleep(5)
277 for permission in self.permissions: 306 for permission in self.permissions:
278 - try: 307 + self.add_permission(
279 - kwargs = { 308 + permission['action'],
280 - 'FunctionName': self.name, 309 + permission['principal'],
281 - 'StatementId': permission['statement_id'], 310 + permission.get('source_arn'),
282 - 'Action': permission['action'], 311 + permission.get('source_account'))
283 - 'Principal': permission['principal']}
284 - source_arn = permission.get('source_arn', None)
285 - if source_arn:
286 - kwargs['SourceArn'] = source_arn
287 - source_account = permission.get('source_account', None)
288 - if source_account:
289 - kwargs['SourceAccount'] = source_account
290 - response = self._lambda_client.call(
291 - 'add_permission', **kwargs)
292 - LOG.debug(response)
293 - except Exception:
294 - LOG.exception('Unable to add permission')
295 312
296 def create(self): 313 def create(self):
297 LOG.info('creating function %s', self.name) 314 LOG.info('creating function %s', self.name)
...@@ -415,15 +432,6 @@ class Function(object): ...@@ -415,15 +432,6 @@ class Function(object):
415 response = None 432 response = None
416 return response 433 return response
417 434
418 - def invoke_asynch(self, data_file):
419 - LOG.debug('_invoke_async %s', data_file)
420 - with open(data_file) as fp:
421 - response = self._lambda_client.call(
422 - 'invoke_async',
423 - FunctionName=self.name,
424 - InvokeArgs=fp)
425 - LOG.debug(response)
426 -
427 def _invoke(self, data, invocation_type): 435 def _invoke(self, data, invocation_type):
428 LOG.debug('invoke %s as %s', self.name, invocation_type) 436 LOG.debug('invoke %s as %s', self.name, invocation_type)
429 response = self._lambda_client.call( 437 response = self._lambda_client.call(
......
...@@ -89,7 +89,8 @@ class Policy(object): ...@@ -89,7 +89,8 @@ class Policy(object):
89 PolicyArn=self.arn) 89 PolicyArn=self.arn)
90 except Exception: 90 except Exception:
91 LOG.exception('Error listing policy versions') 91 LOG.exception('Error listing policy versions')
92 - return response['Versions'] 92 + response = {}
93 + return response.get('Versions', list())
93 94
94 def exists(self): 95 def exists(self):
95 for policy in self._find_all_policies(): 96 for policy in self._find_all_policies():
......