v1

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+keys_local.py

README.md

@@ -0,0 +1,47 @@
+# tweetzotero
+
+tweetzotero is a python automation script that takes a csv file with tweet URLs, fetch all the external URLs from those example, and save those to Zotero.
+
+Prerequisites
+----
+Here's the things that needs to be done before running the script:
+1. Get your Twitter `API Key`, `API Secret`, `Access Token` and `Access Secret` from [here](https://developer.twitter.com/en/portal/dashboard).
+2. Set your default browser to `Google Chrome`. This script will work on this browser for now.
+3. Export your twitter bookmark URLs as `csv` with the chrome extension [Dewey](https://chrome.google.com/webstore/detail/dewey/occohfgiljdagdmklhpplgmcnliljmgi).
+4. Add Zotero extension to Google Chrome from [here](https://chrome.google.com/webstore/detail/zotero-connector/ekhagklcjbdpajgpjgmbionohlpdbjgc?hl=en). While running the script, the zotero app must be running.
+5. `tweetzotero` uses the shortcut `ctrl/cmd + shift + s` on chrome to add the paper to zotero. So, make sure this shortcut actually works on chrome before running the script.
+6. Make sure you have `conda` installed. If not, then install from [here](https://docs.conda.io/en/latest/miniconda.html).
+
+Setup
+----
+
+Here's how you can get up and running:
+```sh
+git clone https://github.com/nahid18/tweetzotero.git
+cd tweetzotero
+conda create --name tweetzotero --file requirements.txt -y
+conda activate tweetzotero
+```
+###### **Now, paste all 4 twitter keys to the `keys.py` file.**
+After that, you can run the script like below.
+
+Usage
+----
+
+Once you have completed everything above, here's how you use the script.
+```sh
+python tweetzotero.py -i <dewey-csv-file-path>
+```
+Example:
+```sh
+python tweetzotero.py -i example.csv
+```
+For Help,
+```sh
+python tweetzotero.py -h
+```
+
+License
+----
+
+MIT

example.csv

@@ -0,0 +1,4 @@
+Name,Tags
+https://twitter.com/amjjbonvin/status/1404339275244740608,
+https://twitter.com/BenjaminWerner_/status/1403752956021948420,
+https://twitter.com/chennifer_jen/status/1403836056840060932,

keys.py

@@ -0,0 +1,4 @@
+api_key = ""
+api_secret = ""
+access_token = ""
+access_token_secret = ""

requirements.txt

@@ -0,0 +1,65 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: linux-64
+_libgcc_mutex=0.1=conda_forge
+_openmp_mutex=4.5=1_gnu
+beautifulsoup4=4.9.3=pyhb0f4dca_0
+blinker=1.4=py_1
+brotlipy=0.7.0=py38h497a2fe_1001
+ca-certificates=2020.10.14=0
+certifi=2020.6.20=py38_0
+cffi=1.14.5=py38ha65f79e_0
+chardet=4.0.0=py38h578d9bd_1
+cryptography=3.4.7=py38ha5dfef3_0
+freetype=2.10.4=h0708190_1
+icu=58.2=he6710b0_3
+idna=2.10=pyh9f0ad1d_0
+jbig=2.1=h7f98852_2003
+jpeg=9d=h36c2ea0_0
+lcms2=2.12=hddcbb42_0
+ld_impl_linux-64=2.35.1=hea4e1c9_2
+lerc=2.2.1=h9c3ff4c_0
+libdeflate=1.7=h7f98852_5
+libffi=3.3=h58526e2_2
+libgcc-ng=9.3.0=h2828fa1_19
+libgomp=9.3.0=h2828fa1_19
+libpng=1.6.37=h21135ba_2
+libstdcxx-ng=9.3.0=h6de172a_19
+libtiff=4.3.0=hf544144_1
+libwebp-base=1.2.0=h7f98852_2
+libxml2=2.9.10=hb55368b_3
+libxslt=1.1.34=hc22bd24_0
+lxml=4.6.1=py38hefd8a0e_0
+lz4-c=1.9.3=h9c3ff4c_0
+ncurses=6.2=h58526e2_4
+oauthlib=3.1.1=pyhd8ed1ab_0
+olefile=0.46=pyh9f0ad1d_1
+openjpeg=2.4.0=hb52868f_1
+openssl=1.1.1k=h7f98852_0
+pillow=8.2.0=py38ha0e1e83_1
+pip=21.1.2=pyhd8ed1ab_0
+pyautogui=0.9.48=py38h32f6830_1
+pycparser=2.20=pyh9f0ad1d_2
+pyjwt=2.1.0=pyhd8ed1ab_0
+pymsgbox=1.0.9=pyh9f0ad1d_0
+pyopenssl=20.0.1=pyhd8ed1ab_0
+pyscreeze=0.1.27=pyhd8ed1ab_0
+pysocks=1.7.1=py38h578d9bd_3
+python=3.8.10=h49503c6_1_cpython
+python-xlib=0.28=pyh9f0ad1d_0
+python_abi=3.8=1_cp38
+pytweening=1.0.3=py_0
+readline=8.1=h46c0cb4_0
+requests=2.25.1=pyhd3deb0d_0
+requests-oauthlib=1.3.0=pyh9f0ad1d_0
+setuptools=49.6.0=py38h578d9bd_3
+six=1.16.0=pyh6c4a22f_0
+soupsieve=2.0.1=py_0
+sqlite=3.35.5=h74cdb3f_0
+tk=8.6.10=h21135ba_1
+tweepy=3.10.0=pyhd8ed1ab_0
+urllib3=1.26.5=pyhd8ed1ab_0
+wheel=0.36.2=pyhd3deb0d_0
+xz=5.2.5=h516909a_1
+zlib=1.2.11=h516909a_1010
+zstd=1.5.0=ha95c52a_0

tweetzotero.py

@@ -0,0 +1,71 @@
+from bs4 import BeautifulSoup
+from keys import *
+import webbrowser
+import pyautogui
+import requests
+import argparse
+import tweepy
+import time
+
+def fetch_urls_in_tweet(api, status_id):
+    status = api.get_status(status_id, tweet_mode="extended")
+    try:
+        content = status.retweeted_status.full_text
+    except AttributeError:  
+        content = status.full_text
+    urls = [i for i in content.split() if i.lower().startswith(("https://", "buff.ly", "t.co", "bit.ly"))]
+    return urls
+
+def request_url(short_url):
+    try:
+        extended = requests.get(short_url, timeout=5)
+        return extended.url
+    except Exception as e: 
+        print(e)
+
+def get_extended_url(api, tweet_list):
+    all_urls = list()
+    all_urls.extend(fetch_urls_in_tweet(api=api, status_id=i) for i in tweet_list)
+    all_urls = [item for sublist in all_urls for item in sublist]
+    extended_urls = [request_url(i) for i in all_urls]
+    save_list(inlist=extended_urls)
+    return extended_urls
+
+def save_list(inlist):
+    url_list = [i for i in inlist if i is not None]
+    with open('all_urls.txt', 'w') as fh:
+        for item in url_list:
+            fh.write("%s\n" % item)
+
+def save_to_zotero(url):
+    webbrowser.open(url)
+    time.sleep(5)
+    pyautogui.hotkey('ctrl', 'shift', 's')
+    time.sleep(10)
+
+
+def run(tweet_ids):
+    auth = tweepy.OAuthHandler(api_key, api_secret)
+    auth.set_access_token(access_token, access_token_secret)
+    api = tweepy.API(auth)
+    extended_urls = get_extended_url(api=api, tweet_list=tweet_ids)
+    for url in extended_urls:
+        try:
+            save_to_zotero(url=url)
+        except Exception as e: 
+            print(e)
+
+
+parser = argparse.ArgumentParser(description='Save Twitter Bookmarked Papers to Zotero')
+parser.add_argument('-i','--input', help='Provide input', required=True)
+args = parser.parse_args()
+
+if args.input:
+    try:
+        with open(args.input, "r") as handle:
+            tweet_ids=[i.split(',')[0].split('/')[-1] for i in handle.readlines()[1:]]
+        run(tweet_ids=tweet_ids)
+    except Exception as e: 
+        print(e)
+else:
+    print('Provide an input file')