Skip to content

Commit b934acd

Browse files
HzANutHzANut
authoredOct 12, 2022
[WIP] add config, sampling rate support, and bump up version (#14)
* add sampling rate and config support - added initialization code to improve the performance, and refactor the sampling to central api - code cleanup - version bump
·
1.2.21.1.0
1 parent a05fbda commit b934acd

File tree

5 files changed

+155
-57
lines changed

5 files changed

+155
-57
lines changed
 

‎.gitignore

+2-2
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ celerybeat-schedule
8787
.env
8888

8989
# virtualenv
90-
.venv/
91-
venv/
90+
.venv*/
91+
venv*/
9292
ENV/
9393

9494
# Spyder project settings

‎moesif_aws_lambda/global_variable.py

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
"""This module is to declare global objects."""
2+
from datetime import datetime
3+
from moesifpythonrequest.app_config.app_config import AppConfig
4+
from moesifapi.moesif_api_client import *
5+
import os
6+
7+
# MoesifAPI Client
8+
global api_client
9+
api_client = None
10+
11+
# App Config class
12+
global app_config
13+
app_config = None
14+
15+
# App Config
16+
global config
17+
config = None
18+
19+
# App Config sampling percentage
20+
global sampling_percentage
21+
sampling_percentage = 100
22+
23+
# App Config eTag
24+
global config_etag
25+
config_etag = None
26+
27+
# App Config last updated time
28+
global last_updated_time
29+
last_updated_time = datetime.utcnow()
30+
31+
# the delta time for refreshing config, if the last update time is greater than this
32+
global refresh_config_time_seconds
33+
refresh_config_time_seconds = 2 * 60
34+
35+
# initialize the config first time on cold start.
36+
print("[moesif] start init - config")
37+
38+
# Initialize the client
39+
if os.environ.get("MOESIF_APPLICATION_ID"):
40+
api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
41+
else:
42+
raise Exception('Moesif Application ID is required in settings')
43+
44+
app_config = AppConfig()
45+
config = app_config.get_config(api_client, True)
46+
47+
print("[moesif] end init - config")

‎moesif_aws_lambda/middleware.py

+103-52
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,18 @@
77
from .client_ip import ClientIp
88
from .update_companies import Company
99
from .update_users import User
10+
from . import global_variable as gv
11+
1012
from datetime import *
1113
import base64
1214
import json
1315
import os
1416
from pprint import pprint
1517
import base64
18+
19+
import random
20+
import math
21+
1622
try:
1723
from urllib import urlencode
1824
except ImportError:
@@ -24,32 +30,40 @@
2430
def get_time_took_in_ms(start_time, end_time):
2531
return (end_time - start_time).total_seconds() * 1000
2632

33+
2734
def start_capture_outgoing(moesif_options):
2835
try:
2936
if moesif_options.get('DEBUG', False):
30-
print('Start capturing outgoing requests')
37+
print('[moesif] Start capturing outgoing requests')
38+
3139
# Start capturing outgoing requests
3240
moesif_options['APPLICATION_ID'] = os.environ["MOESIF_APPLICATION_ID"]
3341
StartCapture().start_capture_outgoing(moesif_options)
42+
43+
if moesif_options.get('DEBUG', False):
44+
print("[moesif] end capturing moesif options")
3445
except Exception as e:
3546
print('Error while starting to capture the outgoing events')
3647
print(e)
48+
return
49+
3750

3851
# Initialized the client
39-
if os.environ["MOESIF_APPLICATION_ID"]:
40-
api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
41-
else:
42-
raise Exception('Moesif Application ID is required in settings')
52+
api_client = gv.api_client
53+
4354

4455
def update_user(user_profile, moesif_options):
4556
User().update_user(user_profile, api_client, moesif_options)
4657

58+
4759
def update_users_batch(user_profiles, moesif_options):
4860
User().update_users_batch(user_profiles, api_client, moesif_options)
4961

62+
5063
def update_company(company_profile, moesif_options):
5164
Company().update_company(company_profile, api_client, moesif_options)
5265

66+
5367
def update_companies_batch(companies_profiles, moesif_options):
5468
Company().update_companies_batch(companies_profiles, api_client, moesif_options)
5569

@@ -70,12 +84,9 @@ def __init__(self, handler):
7084
self.event = None
7185
self.context = None
7286
self.payload_version = None
73-
74-
# Intialized the client
75-
if os.environ.get("MOESIF_APPLICATION_ID"):
76-
self.api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
77-
else:
78-
raise Exception('Moesif Application ID is required in settings')
87+
88+
# Set the client
89+
self.api_client = api_client
7990

8091
def clear_state(self):
8192
"""Function to clear state of local variable"""
@@ -108,16 +119,16 @@ def get_user_id(self, event, context):
108119
username = rc_identity_id
109120
except:
110121
if self.DEBUG:
111-
print("MOESIF can not fetch apiKey from cognitoIdentityId event, setting userId to None.")
122+
print("[moesif] can not fetch apiKey from cognitoIdentityId event, setting userId to None.")
112123
except Exception as e:
113124
if self.DEBUG:
114-
print("MOESIF can not execute identify_user function, please check moesif settings.")
125+
print("[moesif] can not execute identify_user function, please check moesif settings.")
115126
print(e)
116127
end_time_get_user_id = datetime.utcnow()
117128
if self.DEBUG:
118129
print("[moesif] Time took in fetching user id in millisecond - " + str(get_time_took_in_ms(start_time_get_user_id, end_time_get_user_id)))
119130
return username
120-
131+
121132
def get_company_id(self, event, context):
122133
"""Function to fetch CompanyId"""
123134
start_time_get_company_id = datetime.utcnow()
@@ -128,23 +139,23 @@ def get_company_id(self, event, context):
128139
company_id = identify_company(event, context)
129140
except Exception as e:
130141
if self.DEBUG:
131-
print("MOESIF can not execute identify_company function, please check moesif settings.")
142+
print("[moesif] can not execute identify_company function, please check moesif settings.")
132143
print(e)
133144
end_time_get_company_id = datetime.utcnow()
134145
if self.DEBUG:
135146
print("[moesif] Time took in fetching company id in millisecond - " + str(get_time_took_in_ms(start_time_get_company_id, end_time_get_company_id)))
136147
return company_id
137-
148+
138149
def build_uri(self, event, payload_format_version_1_0):
139150

140-
uri = event['headers'].get('X-Forwarded-Proto', event['headers'].get('x-forwarded-proto', 'http')) + '://' + event['headers'].get('Host', event['headers'].get('host', 'localhost'))
141-
151+
uri = event['headers'].get('X-Forwarded-Proto', event['headers'].get('x-forwarded-proto', 'http')) + '://' + event['headers'].get('Host', event['headers'].get('host', 'localhost'))
152+
142153
if payload_format_version_1_0:
143154
uri = uri + event.get('path', '/')
144155
if event.get('multiValueQueryStringParameters', {}):
145-
uri = uri + '?' + urlencode(event['multiValueQueryStringParameters'], doseq=True)
156+
uri = uri + '?' + urlencode(event['multiValueQueryStringParameters'], doseq=True)
146157
elif event.get('queryStringParameters', {}):
147-
uri = uri + '?' + urlencode(event['queryStringParameters'])
158+
uri = uri + '?' + urlencode(event['queryStringParameters'])
148159
else:
149160
uri = uri + event.get('rawPath', '/')
150161
if event.get('rawQueryString', {}):
@@ -211,7 +222,7 @@ def before(self, event, context):
211222
else:
212223
request_verb = event.get('requestContext', {}).get('http', {}).get('method')
213224
if request_verb is None:
214-
print('MOESIF: [before] AWS Lambda trigger must be a Load Balancer or API Gateway See https://docs.aws.amazon.com/lambda/latest/dg/services-alb.html or https://docs.aws.amazon.com/lambda/latest/dg/with-on-demand-https.html.')
225+
print('[moesif] : [before] AWS Lambda trigger must be a Load Balancer or API Gateway See https://docs.aws.amazon.com/lambda/latest/dg/services-alb.html or https://docs.aws.amazon.com/lambda/latest/dg/with-on-demand-https.html.')
215226
self.event = None
216227
self.context = None
217228
self.payload_version = None
@@ -224,23 +235,23 @@ def before(self, event, context):
224235
req_headers = APIHelper.json_deserialize(event['headers'])
225236
except Exception as e:
226237
if self.DEBUG:
227-
print('MOESIF Error while fetching request headers')
238+
print('[moesif] Error while fetching request headers')
228239
print(e)
229240

230241
# Request Time
231242
if self.is_payload_format_version_1_0(self.payload_version):
232243
epoch = event and event.get('requestContext', {}).get('requestTimeEpoch')
233244
else:
234245
epoch = event and event.get('requestContext', {}).get('timeEpoch')
235-
if epoch is not None:
246+
if epoch is not None:
236247
# Dividing by 1000 to convert from ms to seconds and `.0` to preserve millisecond precision
237248
request_time = datetime.utcfromtimestamp(epoch/1000.0)
238249
else:
239250
request_time = datetime.utcnow()
240-
251+
241252
# Request Body
242253
req_body, req_transfer_encoding = self.process_body(event)
243-
254+
244255
# Metadata
245256
start_time_get_metadata = datetime.utcnow()
246257
try:
@@ -258,15 +269,15 @@ def before(self, event, context):
258269
}
259270
except:
260271
if self.DEBUG:
261-
print("MOESIF can not fetch default function_name and request_context from aws context, setting metadata to None.")
272+
print("[moesif] can not fetch default function_name and request_context from aws context, setting metadata to None.")
262273
except Exception as e:
263274
if self.DEBUG:
264-
print("MOESIF can not execute GET_METADATA function, please check moesif settings.")
275+
print("[moesif] can not execute GET_METADATA function, please check moesif settings.")
265276
print(e)
266277
end_time_get_metadata = datetime.utcnow()
267278
if self.DEBUG:
268279
print("[moesif] Time took in fetching metadata in millisecond - " + str(get_time_took_in_ms(start_time_get_metadata, end_time_get_metadata)))
269-
280+
270281
# User Id
271282
start_time_identify_user = datetime.utcnow()
272283
self.user_id = self.get_user_id(event, context)
@@ -294,10 +305,10 @@ def before(self, event, context):
294305
self.session_token = rc_api_key
295306
except KeyError:
296307
if self.DEBUG:
297-
print("MOESIF can not fetch apiKey from aws event, setting session_token to None.")
308+
print("[moesif] can not fetch apiKey from aws event, setting session_token to None.")
298309
except Exception as e:
299310
if self.DEBUG:
300-
print("MOESIF can not execute GET_SESSION_TOKEN function, please check moesif settings.")
311+
print("[moesif] can not execute GET_SESSION_TOKEN function, please check moesif settings.")
301312
print(e)
302313

303314
# Api Version
@@ -312,12 +323,12 @@ def before(self, event, context):
312323
api_version = context.function_version
313324
except KeyError:
314325
if self.DEBUG:
315-
print("MOESIF can not fetch default function_version from aws context, setting api_version to None.")
326+
print("[moesif] can not fetch default function_version from aws context, setting api_version to None.")
316327
except Exception as e:
317328
if self.DEBUG:
318-
print("MOESIF can not execute GET_API_VERSION function, please check moesif settings.")
329+
print("[moesif] can not execute GET_API_VERSION function, please check moesif settings.")
319330
print(e)
320-
331+
321332
# IpAddress
322333
if self.is_payload_format_version_1_0(self.payload_version):
323334
ip_address = event.get('requestContext', {}).get('identity', {}).get('sourceIp', None)
@@ -339,11 +350,12 @@ def before(self, event, context):
339350
print("[moesif] Time took before the handler is invoked in millisecond - " + str(get_time_took_in_ms(start_time_before_handler_function, end_time_before_handler_function)))
340351
# Return event, context
341352
return event, context
342-
353+
343354
def after(self, retval):
344355
"""This function runs after the handler is invoked, is passed the response and must return an response too."""
345-
356+
346357
start_time_after_handler_function = datetime.utcnow()
358+
event_send = None
347359
if self.event is not None:
348360
# Response body
349361
resp_body, resp_transfer_encoding = self.process_body(retval)
@@ -362,27 +374,27 @@ def after(self, retval):
362374
company_id = self.company_id,
363375
session_token = self.session_token,
364376
metadata = self.metadata)
365-
377+
366378
# Mask Event Model
367379
try:
368380
mask_event_model = self.moesif_options.get('MASK_EVENT_MODEL', None)
369381
if mask_event_model is not None:
370382
event_model = mask_event_model(event_model)
371-
except:
383+
except Exception as e:
372384
if self.DEBUG:
373-
print("MOESIF Can not execute MASK_EVENT_MODEL function. Please check moesif settings.")
385+
print("[moesif] Can not execute MASK_EVENT_MODEL function. Please check moesif settings.", e)
374386

375387
# Skip Event
376388
try:
377389
skip_event = self.moesif_options.get('SKIP', None)
378390
if skip_event is not None:
379391
if skip_event(self.event, self.context):
380392
if self.DEBUG:
381-
print('MOESIF Skip sending event to Moesif')
393+
print('[moesif] Skip sending event to Moesif')
382394
return retval
383-
except:
395+
except Exception as e:
384396
if self.DEBUG:
385-
print("MOESIF Having difficulty executing skip_event function. Please check moesif settings.")
397+
print("[moesif] Having difficulty executing skip_event function. Please check moesif settings.", e)
386398

387399
# Add direction field
388400
event_model.direction = "Incoming"
@@ -391,17 +403,56 @@ def after(self, retval):
391403
if self.DEBUG:
392404
print('Moesif Event Model:')
393405
print(json.dumps(self.event))
394-
395-
if self.DEBUG:
396-
start_time_sending_event_w_rsp = datetime.utcnow()
397-
event_send = self.api_client.create_event(event_model)
398-
end_time_sending_event_w_rsp = datetime.utcnow()
399-
if self.DEBUG:
400-
print("[moesif] Time took in sending event to moesif in millisecond - " + str(get_time_took_in_ms(start_time_sending_event_w_rsp, end_time_sending_event_w_rsp)))
401-
print('[moesif] Event Sent successfully ' + str(event_send))
402-
else:
403-
self.api_client.create_event(event_model)
404-
406+
407+
# Sampling Rate
408+
try:
409+
random_percentage = random.random() * 100
410+
gv.sampling_percentage = gv.app_config.get_sampling_percentage(
411+
event_model,
412+
gv.config,
413+
self.user_id,
414+
self.company_id,
415+
)
416+
417+
if gv.sampling_percentage >= random_percentage:
418+
event_model.weight = 1 if gv.sampling_percentage == 0 else math.floor(
419+
100 / gv.sampling_percentage)
420+
421+
if self.DEBUG:
422+
start_time_sending_event_w_rsp = datetime.utcnow()
423+
event_send = self.api_client.create_event(event_model)
424+
end_time_sending_event_w_rsp = datetime.utcnow()
425+
print("[moesif] sampling_percentage" + str(
426+
gv.sampling_percentage) + " and random percentage: " + str(random_percentage))
427+
print("[moesif] Time took in sending event to moesif in millisecond - " + str(
428+
get_time_took_in_ms(start_time_sending_event_w_rsp, end_time_sending_event_w_rsp)))
429+
print('[moesif] Event Sent successfully ' + str(event_send))
430+
431+
else:
432+
if datetime.utcnow() > gv.last_updated_time + timedelta(seconds=gv.refresh_config_time_seconds):
433+
event_send = self.api_client.create_event(event_model)
434+
else:
435+
self.api_client.create_event(event_model)
436+
437+
try:
438+
# Check if we need to update config
439+
new_config_etag = event_send['x-moesif-config-etag']
440+
if gv.config_etag is None or (gv.config_etag != new_config_etag):
441+
gv.config_etag = new_config_etag
442+
gv.config = gv.app_config.get_config(self.api_client, self.DEBUG)
443+
except (KeyError, TypeError, ValueError) as ex:
444+
# ignore the error because "event_send" is not set in non-blocking call
445+
pass
446+
finally:
447+
gv.last_updated_time = datetime.utcnow()
448+
449+
else:
450+
if self.DEBUG:
451+
print("Skipped Event due to sampling percentage: " + str(
452+
gv.sampling_percentage) + " and random percentage: " + str(random_percentage))
453+
except Exception as ex:
454+
print("[moesif] Error when fetching sampling rate from app config", ex)
455+
405456
end_time_after_handler_function = datetime.utcnow()
406457
if self.DEBUG:
407458
print("[moesif] Time took after the handler is invoked in millisecond - " + str(get_time_took_in_ms(start_time_after_handler_function, end_time_after_handler_function)))

‎requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
moesifapi==1.4.0
22
lambda_decorators==0.3.0
3-
moesifpythonrequest==0.2.0
3+
moesifpythonrequest==0.3.0

‎setup.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
# Versions should comply with PEP440. For a discussion on single-sourcing
2929
# the version across setup.py and the project code, see
3030
# https://packaging.python.org/en/latest/single_source_version.html
31-
version='1.0.10',
31+
version='1.1.0',
3232

3333
description='Moesif Middleware to automatically log API calls from AWS Lambda functions',
3434
long_description=long_description,
@@ -84,7 +84,7 @@
8484
# your project is installed. For an analysis of "install_requires" vs pip's
8585
# requirements files see:
8686
# https://packaging.python.org/en/latest/requirements.html
87-
install_requires=['moesifapi>=1.4.0', 'lambda_decorators', 'moesifpythonrequest>=0.2.0'],
87+
install_requires=['moesifapi>=1.4.0', 'lambda_decorators', 'moesifpythonrequest>=0.3.0'],
8888

8989
# List additional groups of dependencies here (e.g. development
9090
# dependencies). You can install these using the following syntax,

0 commit comments

Comments
 (0)
Please sign in to comment.