Ok, I've got the daily stuff up and running and linked to a spreadsheet. Cool stuff!
(get-twitter-likes) user1@jh-ltm2 get-twitter-likes % python get_likes_from_json_to_csv.py -pu -o data/likes1.csv
Namespace(input='data/like.js', output='data/likes1.csv', format='raw', parse_urls=True, save_json_col=False)
Reading likes from /Users/user1/Projects/get-twitter-likes/data/like.js
Downloading likes detailed data from Twitter API and parsing their URLs
Traceback (most recent call last):
File "/Users/user1/Projects/get-twitter-likes/get_likes_from_json_to_csv.py", line 28, in <module>
likes = get_likes_from_json(input_file=input_file,
File "/Users/user1/Projects/get-twitter-likes/src/core.py", line 166, in get_likes_from_json
tweets_lists = parallel(lookup_and_parse_tweets,
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastcore/parallel.py", line 106, in parallel
return L(r)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastcore/foundation.py", line 97, in __call__
return super().__call__(x, *args, **kwargs)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastcore/foundation.py", line 105, in __init__
items = listify(items, *rest, use_list=use_list, match=match)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastcore/basics.py", line 56, in listify
elif is_iter(o): res = list(o)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastprogress/fastprogress.py", line 47, in __iter__
raise e
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastprogress/fastprogress.py", line 41, in __iter__
for i,o in enumerate(self.gen):
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 609, in result_iterator
yield fs.pop().result()
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 446, in result
return self.__get_result()
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
raise self._exception
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/fastcore/parallel.py", line 54, in _call
return g(item)
File "/Users/user1/Projects/get-twitter-likes/src/core.py", line 183, in lookup_and_parse_tweets
parsed_tweets = parse_tweets(statuses, output_format=output_format, parse_urls=parse_urls)
File "/Users/user1/Projects/get-twitter-likes/src/core.py", line 190, in parse_tweets
list_of_lists_of_parsed_tweets = parallel_map(parse_tweet,
File "/Users/user1/Projects/get-twitter-likes/src/utils.py", line 20, in parallel_map
return list(result)
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 609, in result_iterator
yield fs.pop().result()
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 439, in result
return self.__get_result()
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
raise self._exception
File "/usr/local/Cellar/[email protected]/3.9.14/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/Users/user1/Projects/get-twitter-likes/src/core.py", line 85, in parse_tweet
title_description = get_url_title_description(url, parse_url=parse_url)
File "/Users/user1/Projects/get-twitter-likes/src/scraping.py", line 16, in get_url_title_description
response = session.get(url, timeout=3)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests/sessions.py", line 555, in get
return self.request('GET', url, **kwargs)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/core.py", line 132, in request
response = super(CachedSession, self).request(
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/core.py", line 109, in send
return send_request_and_cache_response()
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/core.py", line 99, in send_request_and_cache_response
self.cache.save_response(cache_key, response)
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/backends/base.py", line 46, in save_response
self.responses[key] = self.reduce_response(response), datetime.utcnow()
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/backends/storage/dbdict.py", line 165, in __setitem__
super(DbPickleDict, self).__setitem__(key,
File "/Users/user1/.local/share/virtualenvs/get-twitter-likes-b1tRvag8/lib/python3.9/site-packages/requests_cache/backends/storage/dbdict.py", line 129, in __setitem__
con.execute("insert or replace into `%s` (key,value) values (?,?)" %
sqlite3.OperationalError: database is locked