iterative development, token implemented

Author: WaseeA

src/py_scripts/script.py

@@ -18,7 +18,10 @@
 
 def sentence_to_embeddings(query):
     # unpack the models
-    print(query)
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    model = BertModel.from_pretrained('bert-base-uncased')
+    tokens = tokenizer.tokenize(query)
+    print(tokens)
 
 if __name__ == "__main__":
     # The query is passed as the second argument
@@ -34,4 +37,5 @@ def sentence_to_embeddings(query):
 # SELECT run_py('/data/trino/src/py_scripts/script.py') AS result;
 # SELECT square(4);
 # docker exec -it trino-nlp-embeddings bash
-# docker cp src/py_scripts/requirements.txt trino-nlp-embeddings:/data/trino/src/py_scripts/requirements.txt
+# docker cp src/py_scripts/requirements.txt trino-nlp-embeddings:/data/trino/src/py_scripts/requirements.txt
+# docker cp src/py_scripts/script.py trino-nlp-embeddings:/data/trino/src/py_scripts/script.py