unimore/pds-2021-g2-nest
unimore/pds-2021-g2-nest
1
Fork 0
mirror of https://github.com/pds-nest/nest.git synced 2024-11-25 06:24:19 +00:00
Code Activity

Riordinato codice e aggiunta la conversione dei metri in kilometri nella specifica delle condizioni di tipo coordinates

Browse source
This commit is contained in:
g.minoccari 2021-05-17 10:17:00 +02:00
parent 09fbbc8ae4
commit bdcca12de2
1 changed files with 16 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
nest_crawler/__main__.py
View file

@ -1,8 +1,6 @@
from nest_backend.app import app from nest_backend.app import app
from nest_backend.database import * from nest_backend.database import *
import tweepy as tw import tweepy as tw
import nltk
from nltk.corpus import stopwords
from datetime import datetime from datetime import datetime
ext.init_app(app=app) ext.init_app(app=app)
@ -13,31 +11,30 @@ def authenticate():
c_s = "438cmYrl5xqaX2W7I2Bf5A9nF1pN5VtM9f77WYQnAXg1BwKJ27" c_s = "438cmYrl5xqaX2W7I2Bf5A9nF1pN5VtM9f77WYQnAXg1BwKJ27"
a_t = "1380217745732689921-8gCfr8Zx9YHKvo4OVP3HAr3kfMRkgz" a_t = "1380217745732689921-8gCfr8Zx9YHKvo4OVP3HAr3kfMRkgz"
a_t_s = "jGOlgTs1i1itGMxDxAqFEDnv7QAui772n9hGxeSIKcwzS" a_t_s = "jGOlgTs1i1itGMxDxAqFEDnv7QAui772n9hGxeSIKcwzS"
b_t = "AAAAAAAAAAAAAAAAAAAAANjpOQEAAAAAYFKrqByk6WTMEwIYUJLrVnPeQuo%3DJUJUW2mFq9n20A7BtIRMT24PPFg1NXVtetrfp0tH1NkDFFNGo5"
auth = tw.OAuthHandler(c_k, c_s) auth = tw.OAuthHandler(c_k, c_s)
auth.set_access_token(a_t, a_t_s) auth.set_access_token(a_t, a_t_s)
api = tw.API(auth, wait_on_rate_limit=True) api = tw.API(auth, wait_on_rate_limit=True)
#client = tw.Client(b_t, c_k, c_s, a_t, a_t_s, wait_on_rate_limit=True); # client = tw.Client(b_t, c_k, c_s, a_t, a_t_s, wait_on_rate_limit=True);
return api return api
def search_repo_conditions(repository_id): def search_repo_conditions(repository_id):
api = authenticate() api = authenticate()
geocode="44.3591600,11.7132000,20km" geocode = "44.3591600,11.7132000,20km"
repo = Repository.query.filter_by(id=repository_id).first() repo = Repository.query.filter_by(id=repository_id).first()
if(repo == None): if repo is None:
print("Non esiste una repository con questo id") print("Non esiste una repository con questo id")
return False return False
conditions = [use for use in repo.conditions] conditions = [use for use in repo.conditions]
evaluation_mode = repo.evaluation_mode evaluation_mode = repo.evaluation_mode
conditions_type = dict() conditions_type = dict()
#Dividing condition into condition types # Dividing condition into condition types
for condition in conditions: for condition in conditions:
# print(condition.id) # print(condition.id)
if condition.type not in conditions_type.keys(): if condition.type not in conditions_type.keys():
conditions_type[condition.type]=[condition.content] conditions_type[condition.type] = [condition.content]
else: else:
conditions_type[condition.type].append(condition.content) conditions_type[condition.type].append(condition.content)
@ -55,15 +52,15 @@ def search_repo_conditions(repository_id):
queryString += ("#" + condition_content + " " + queryConjunction + " ") queryString += ("#" + condition_content + " " + queryConjunction + " ")
if ConditionType.coordinates in conditions_type.keys(): if ConditionType.coordinates in conditions_type.keys():
if evaluation_mode == ConditionMode.all_and: if evaluation_mode == ConditionMode.all_and:
if len(conditions_type[ConditionType.coordinates])==1: if len(conditions_type[ConditionType.coordinates]) == 1:
coordinates_tweet = conditions_type[ConditionType.coordinates][0].split() coordinates_tweet = conditions_type[ConditionType.coordinates][0].split()
coordinates_string = coordinates_tweet[2]+","+coordinates_tweet[3]+","+coordinates_tweet[1]+"km" coordinates_string = coordinates_tweet[2] + "," + coordinates_tweet[3] + "," + str(float(coordinates_tweet[1])/1000) + "km"
else: else:
return None return None
elif evaluation_mode == ConditionMode.all_or: elif evaluation_mode == ConditionMode.all_or:
for condition_content in conditions_type[ConditionType.coordinates]: for condition_content in conditions_type[ConditionType.coordinates]:
coordinates_tweet = condition_content.split() coordinates_tweet = condition_content.split()
coordinates_string = coordinates_tweet[2]+","+coordinates_tweet[3]+","+coordinates_tweet[1]+"km" coordinates_string = coordinates_tweet[2] + "," + coordinates_tweet[3] + "," + str(float(coordinates_tweet[1])/1000) + "km"
print(coordinates_string) print(coordinates_string)
for tweet in tw.Cursor(method=api.search, geocode=coordinates_string).items(10): for tweet in tw.Cursor(method=api.search, geocode=coordinates_string).items(10):
tweetsFound.append(tweet) tweetsFound.append(tweet)
@ -72,23 +69,25 @@ def search_repo_conditions(repository_id):
queryString += ("from:" + condition_content + " " + queryConjunction + " ") queryString += ("from:" + condition_content + " " + queryConjunction + " ")
if ConditionType.time in conditions_type.keys(): if ConditionType.time in conditions_type.keys():
for condition_content in conditions_type[ConditionType.time]: for condition_content in conditions_type[ConditionType.time]:
if condition_content[0]=='<': if condition_content[0] == '<':
queryString += ("until:" + condition_content + " " + queryConjunction + " ") queryString += ("until:" + condition_content + " " + queryConjunction + " ")
elif condition_content[0]=='>': elif condition_content[0] == '>':
queryString += ("since:" + condition_content + " " + queryConjunction + " ") queryString += ("since:" + condition_content + " " + queryConjunction + " ")
queryString = queryString[:-len(queryConjunction)-1] queryString = queryString[:-len(queryConjunction) - 1]
print(queryString) print(queryString)
if(evaluation_mode == ConditionMode.all_or): if evaluation_mode == ConditionMode.all_or:
for tweet in tw.Cursor(method=api.search, q=queryString).items(10): for tweet in tw.Cursor(method=api.search, q=queryString).items(10):
tweetsFound.append(tweet) tweetsFound.append(tweet)
print(tweet.user.name + ' : ' + tweet.text) print(tweet.user.name + ' : ' + tweet.text)
elif(evaluation_mode==ConditionMode.all_and): elif evaluation_mode == ConditionMode.all_and:
for tweet in tw.Cursor(method=api.search, q=queryString, geocode=coordinates_string).items(10): for tweet in tw.Cursor(method=api.search, q=queryString, geocode=coordinates_string).items(10):
tweetsFound.append(tweet) tweetsFound.append(tweet)
print(tweet.user.name + ' : ' + tweet.text) print(tweet.user.name + ' : ' + tweet.text)
for tweet in tweetsFound: for tweet in tweetsFound:
if not Tweet.query.filter_by(snowflake=str(tweet.id)).all(): if not Tweet.query.filter_by(snowflake=str(tweet.id)).all():
tweetDB = Tweet(snowflake=tweet.id, content=tweet.text, location=tweet.geo.coordinate.coordinates if tweet.geo is not None else "", insert_time=str(datetime.now())) tweetDB = Tweet(snowflake=tweet.id, content=tweet.text,
location=tweet.geo.coordinate.coordinates if tweet.geo is not None else "",
insert_time=str(datetime.now()))
ext.session.add(tweetDB) ext.session.add(tweetDB)
if not Composed.query.filter_by(snowflake=str(tweet.id), rid=repository_id).all(): if not Composed.query.filter_by(snowflake=str(tweet.id), rid=repository_id).all():
composed = Composed(rid=repository_id, snowflake=tweet.id) composed = Composed(rid=repository_id, snowflake=tweet.id)
@ -98,7 +97,5 @@ def search_repo_conditions(repository_id):
if __name__ == "__main__": if __name__ == "__main__":
search_repo_conditions(16) search_repo_conditions(16)
#print(stopwords.words('italian'))
with app.app_context(): with app.app_context():
ext.create_all(app=app) ext.create_all(app=app)
#start_exploring()