Skip to content

Commit 6be3345

Browse files
Liam-DriscollLiam-Driscoll
authored
Merge pull request #58 from UBC-CIC/security
Remove logging of sensitive information
2 parents ae70ffb + f613885 commit 6be3345

File tree

4 files changed

+0
-63
lines changed

4 files changed

+0
-63
lines changed

‎back_end/cdk/glue/scripts/patents-etl/fetchEpoPatents.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,9 @@
3838
def authorization():
3939
# encode the keys into base64 representation
4040
combined_string = f"{consumer_key}:{consumer_secret_key}"
41-
print(combined_string)
4241

4342
b = base64.b64encode(bytes(combined_string, 'utf-8')) # bytes
4443
base64_key = b.decode('utf-8')
45-
print(base64_key)
4644

4745
# request access token
4846
headers = {"Authorization": base64_key,
@@ -51,7 +49,6 @@ def authorization():
5149
response = requests.post(
5250
url=ACCESS_AUTH_URL, headers=headers, data=payload)
5351
access_token = response.json()["access_token"]
54-
# print(response.json())
5552
return access_token
5653

5754

@@ -279,13 +276,6 @@ def main(argv):
279276
print(
280277
f"API fetching process exited early. Saving entries we currently have into s3.")
281278
finally:
282-
# datetime object containing current date and time
283-
# now = datetime.now()
284-
# print("now =", now)
285-
# # dd/mm/YY H:M:S
286-
# dt_string = now.strftime("%d-%m-%Y_%H-%M-%S")
287-
# print("date and time =", dt_string)
288-
289279
df = pd.DataFrame(col_dict)
290280
total_patent_count = len(df.index)
291281

‎back_end/cdk/glue/scripts/patents-etl/fetchEquivalentEpoPatents.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,9 @@
3838
def authorization():
3939
# encode the keys into base64 representation
4040
combined_string = f"{consumer_key}:{consumer_secret_key}"
41-
print(combined_string)
4241

4342
b = base64.b64encode(bytes(combined_string, 'utf-8')) # bytes
4443
base64_key = b.decode('utf-8')
45-
print(base64_key)
4644

4745
# request access token
4846
headers = {"Authorization": base64_key,
@@ -51,7 +49,6 @@ def authorization():
5149
response = requests.post(
5250
url=ACCESS_AUTH_URL, headers=headers, data=payload)
5351
access_token = response.json()["access_token"]
54-
# print(response.json())
5552
return access_token
5653

5754

@@ -106,7 +103,6 @@ def fetch_all_equivalent_patent_data():
106103
applicants = [] # list of applicant names
107104
publication_date = ""
108105
inventors = [] # list of inventor names
109-
print(document)
110106
family_number = document['exchange-documents']['exchange-document']['@family-id']
111107
cpc = [] # list of cpc numbers
112108
country_code = ""
@@ -216,13 +212,6 @@ def main(argv):
216212
print(
217213
f"API fetching process exited early. Saving entries we currently have into s3.")
218214
finally:
219-
# datetime object containing current date and time
220-
# now = datetime.now()
221-
# print("now =", now)
222-
# # dd/mm/YY H:M:S
223-
# dt_string = now.strftime("%d-%m-%Y_%H-%M-%S")
224-
# print("date and time =", dt_string)
225-
226215
df = pd.DataFrame(col_dict)
227216
df = df[(df.country_code.str.contains("US")) | (df.country_code.str.contains("CA"))]
228217
total_patent_count = len(df.index)

‎back_end/cdk/glue/scripts/researchers-etl/updatePublications.py‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -125,17 +125,12 @@ def process_researcher_data(cursor, local_database_data, instoken, apikey):
125125
except Exception as e:
126126
log(f"Error fetching data from Scopus API: {e}")
127127
continue
128-
129-
print(author_data_array)
130128

131129
for index, author_data in enumerate(author_data_array):
132130
if author_data['@status'] == 'not_found':
133131
log("Add Researcher To List Of Issue Researchers")
134132
else:
135133
researcher_id, num_documents_scopus, h_index = extract_researcher_info(author_data)
136-
log(researcher_id)
137-
log(num_documents_scopus)
138-
log(h_index)
139134
update_h_index(cursor, researcher_id, h_index)
140135

141136
local_database_data_num_documents = local_database_data[index + index][1]
@@ -282,7 +277,6 @@ def handlePublicationInDatabase(publication, author_id, cursor, connection):
282277

283278
def handlePublicationNotInDatabase(publication, author_id):
284279
try:
285-
log(publication)
286280
author_ids = []
287281
author_names = []
288282

@@ -506,8 +500,6 @@ def store_keywords(author_id, publications, cursor):
506500
cursor.execute(query, (keywords_string, author_id))
507501
except Exception as e:
508502
log(f"Error in store_keywords: {e}")
509-
log(author_id)
510-
log(keywords_string)
511503

512504
def updateResearchers(researchersToUpdateArray, instoken, apikey, connection, cursor):
513505
"""
@@ -556,7 +548,6 @@ def main():
556548
with psycopg2.connect(user=credentials['username'], password=credentials['password'], host=credentials['host'], database=credentials['db']) as connection:
557549
with connection.cursor() as cursor:
558550
researcherArray = createListOfResearchersToUpdate()
559-
log(f"Finished Creating List Of Researchers To Update. List of researchers is: {researcherArray}")
560551

561552
NumberOfPublicationsUpdate = updateResearchers(researcherArray, instoken, apikey, connection, cursor)
562553
log("Finished Updating Researchers")
@@ -580,15 +571,3 @@ def main():
580571
log(f"DMS Glue Job started with Run ID: "+str(response['JobRunId']))
581572

582573
main()
583-
584-
# instoken = ssm_client.get_parameter(Name='/service/elsevier/api/user_name/instoken', WithDecryption=True)
585-
# apikey = ssm_client.get_parameter(Name='/service/elsevier/api/user_name/key', WithDecryption=True)
586-
587-
# authorArray = ["55765887300"]
588-
589-
# url = 'https://api.elsevier.com/content/author'
590-
# headers = {'Accept': 'application/json', 'X-ELS-APIKey': apikey['Parameter']['Value'], 'X-ELS-Insttoken': instoken['Parameter']['Value']}
591-
# params = {'field': 'document-count,h-index', 'author_id': authorArray}
592-
593-
# response = requests.get(url, headers=headers, params=params)
594-
# log(response)

‎back_end/cdk/lib/updatePublicationsImage/updatePublications.py‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -125,17 +125,12 @@ def process_researcher_data(cursor, local_database_data, instoken, apikey):
125125
except Exception as e:
126126
log(f"Error fetching data from Scopus API: {e}")
127127
continue
128-
129-
print(author_data_array)
130128

131129
for index, author_data in enumerate(author_data_array):
132130
if author_data['@status'] == 'not_found':
133131
log("Add Researcher To List Of Issue Researchers")
134132
else:
135133
researcher_id, num_documents_scopus, h_index = extract_researcher_info(author_data)
136-
log(researcher_id)
137-
log(num_documents_scopus)
138-
log(h_index)
139134
update_h_index(cursor, researcher_id, h_index)
140135

141136
local_database_data_num_documents = local_database_data[index + index][1]
@@ -282,7 +277,6 @@ def handlePublicationInDatabase(publication, author_id, cursor, connection):
282277

283278
def handlePublicationNotInDatabase(publication, author_id):
284279
try:
285-
log(publication)
286280
author_ids = []
287281
author_names = []
288282

@@ -506,8 +500,6 @@ def store_keywords(author_id, publications, cursor):
506500
cursor.execute(query, (keywords_string, author_id))
507501
except Exception as e:
508502
log(f"Error in store_keywords: {e}")
509-
log(author_id)
510-
log(keywords_string)
511503

512504
def updateResearchers(researchersToUpdateArray, instoken, apikey, connection, cursor):
513505
"""
@@ -556,7 +548,6 @@ def main():
556548
with psycopg2.connect(user=credentials['username'], password=credentials['password'], host=credentials['host'], database=credentials['db']) as connection:
557549
with connection.cursor() as cursor:
558550
researcherArray = createListOfResearchersToUpdate()
559-
log(f"Finished Creating List Of Researchers To Update. List of researchers is: {researcherArray}")
560551

561552
NumberOfPublicationsUpdate = updateResearchers(researcherArray, instoken, apikey, connection, cursor)
562553
log("Finished Updating Researchers")
@@ -580,15 +571,3 @@ def main():
580571
log(f"DMS Glue Job started with Run ID: "+str(response['JobRunId']))
581572

582573
main()
583-
584-
# instoken = ssm_client.get_parameter(Name='/service/elsevier/api/user_name/instoken', WithDecryption=True)
585-
# apikey = ssm_client.get_parameter(Name='/service/elsevier/api/user_name/key', WithDecryption=True)
586-
587-
# authorArray = ["55765887300"]
588-
589-
# url = 'https://api.elsevier.com/content/author'
590-
# headers = {'Accept': 'application/json', 'X-ELS-APIKey': apikey['Parameter']['Value'], 'X-ELS-Insttoken': instoken['Parameter']['Value']}
591-
# params = {'field': 'document-count,h-index', 'author_id': authorArray}
592-
593-
# response = requests.get(url, headers=headers, params=params)
594-
# log(response)

0 commit comments

Comments
 (0)