From 1f2006c0792e926104a23267da068f863f43db1f Mon Sep 17 00:00:00 2001
From: "@uni-chiara" <258727@studenti.unimore.it>
Date: Sat, 22 May 2021 11:13:46 +0200
Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8C=8D=20Translated=20UI?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
nest_frontend/LocalizationStrings.js | 9 +++++++++
nest_frontend/components/interactive/BoxFilterHashtag.js | 4 ++--
nest_frontend/components/interactive/BoxFilters.js | 3 +--
3 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/nest_frontend/LocalizationStrings.js b/nest_frontend/LocalizationStrings.js
index 8b6b05c..d910b7b 100644
--- a/nest_frontend/LocalizationStrings.js
+++ b/nest_frontend/LocalizationStrings.js
@@ -29,6 +29,8 @@ export default {
byHashtag: "hashtag",
byUser: "utente",
byTimePeriod: "arco di tempo",
+ byContents: "contenuti",
+ byHasPlace: "localizzazione attiva?",
timeBefore: "Prima",
timeAfter: "Dopo",
conditions: "Condizioni",
@@ -99,6 +101,7 @@ export default {
imgTweetsPerc: "% di tweet con immagine",
postUniq: "Totale utenti che hanno postato",
postPop: "Utente più attivo",
+ filters: "Filtri",
},
// 🇬🇧
en: {
@@ -119,6 +122,8 @@ export default {
byHashtag: "hashtag",
byUser: "user",
byTimePeriod: "time period",
+ byContents: "contents",
+ byHasPlace: "is localized?",
timeBefore: "Before",
timeAfter: "After",
conditions: "Conditions",
@@ -189,6 +194,7 @@ export default {
imgTweetsPerc: "% of tweets with image",
postUniq: "Unique posters",
postPop: "Most active poster",
+ filters: "Filters",
},
// 🇫🇮
fi: {
@@ -209,6 +215,8 @@ export default {
byHashtag: "hashtagin mukaan",
byUser: "käyttäjän mukaan",
byTimePeriod: "aikajakson mukaan",
+ byContents: "sisältö",
+ byHasPlace: "on paikallistettu?",
timeBefore: "Ennen",
timeAfter: "Jälkeen",
conditions: "Ehdot",
@@ -279,5 +287,6 @@ export default {
imgTweetsPerc: "% twiiteistä, joissa on kuva",
postUniq: "Ainutkertaiset käyttäjät",
postPop: "Aktiivisimmat käyttäjät",
+ filters: "Suodattimet",
},
}
diff --git a/nest_frontend/components/interactive/BoxFilterHashtag.js b/nest_frontend/components/interactive/BoxFilterHashtag.js
index 76aba03..d3717fe 100644
--- a/nest_frontend/components/interactive/BoxFilterHashtag.js
+++ b/nest_frontend/components/interactive/BoxFilterHashtag.js
@@ -22,9 +22,9 @@ export default function BoxFilterHashtag({ ...props }) {
{strings.searchBy}
-
+
- {strings.byTimePeriod}
+ {strings.byHashtag}
}
{...props}
diff --git a/nest_frontend/components/interactive/BoxFilters.js b/nest_frontend/components/interactive/BoxFilters.js
index 0745124..fc942c8 100644
--- a/nest_frontend/components/interactive/BoxFilters.js
+++ b/nest_frontend/components/interactive/BoxFilters.js
@@ -18,9 +18,8 @@ export default function BoxFilters({ ...props }) {
const badges = filters.map((filter, pos) => )
- // TODO: localize this
return (
-
+
{badges}
)
From efe05351a2c343f4a54bf2c178a4309a7b07695b Mon Sep 17 00:00:00 2001
From: "@uni-chiara" <258727@studenti.unimore.it>
Date: Sat, 22 May 2021 11:28:17 +0200
Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9E=20Fixed=20broken=20icon=20that?=
=?UTF-8?q?=20crashed=20everything?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
nest_frontend/components/interactive/BoxFilterHashtag.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nest_frontend/components/interactive/BoxFilterHashtag.js b/nest_frontend/components/interactive/BoxFilterHashtag.js
index d3717fe..ac7b53f 100644
--- a/nest_frontend/components/interactive/BoxFilterHashtag.js
+++ b/nest_frontend/components/interactive/BoxFilterHashtag.js
@@ -1,6 +1,6 @@
import React from "react"
import BoxFull from "../base/BoxFull"
-import { faClock } from "@fortawesome/free-solid-svg-icons"
+import { faHashtag } from "@fortawesome/free-solid-svg-icons"
import useRepositoryViewer from "../../hooks/useRepositoryViewer"
import useStrings from "../../hooks/useStrings"
import { HashtagFilter } from "../../utils/Filter"
From 6f88f3f5af2ea2c07ce2884e3e141b7bc72b59d4 Mon Sep 17 00:00:00 2001
From: "g.minoccari"
Date: Sat, 22 May 2021 17:00:22 +0200
Subject: [PATCH 3/3] Aggiunto il salvataggio di immagini se presenti nel tweet
salvato, e il salvataggio della data e ora al quale il tweet e' stato postato
---
nest_crawler/__main__.py | 53 +++++++++++++++++++++++++++-------------
1 file changed, 36 insertions(+), 17 deletions(-)
diff --git a/nest_crawler/__main__.py b/nest_crawler/__main__.py
index a802683..468a3ce 100644
--- a/nest_crawler/__main__.py
+++ b/nest_crawler/__main__.py
@@ -53,7 +53,7 @@ def search_repo_conditions(repository_id):
if ConditionType.coordinates in conditions_type.keys():
if evaluation_mode == ConditionMode.all_and:
if len(conditions_type[ConditionType.coordinates]) == 1:
- coordinates_tweet = conditions_type[ConditionType.coordinates][0].split()
+ coordinates_tweet = conditions_type[ConditionType.coordinates][0].content.split()
coordinates_string = coordinates_tweet[2] + "," + coordinates_tweet[3] + "," + str(float(coordinates_tweet[1])/1000) + "km"
else:
return None
@@ -64,11 +64,19 @@ def search_repo_conditions(repository_id):
print(coordinates_string)
for tweet in tw.Cursor(method=api.search, q="", geocode=coordinates_string).items(10):
if not Tweet.query.filter_by(snowflake=str(tweet.id)).all():
+ image_url_list = ''
+ if 'media' in tweet.entities.keys():
+ for url in tweet.entities['media']:
+ image_url_list += (url['media_url_https'] + '|')
+ image_url_list = image_url_list[:-1]
+ else:
+ image_url_list = None
+
tweetDB = Tweet(snowflake=tweet.id, content=tweet.text,
location=tweet.geo['coordinates'] if tweet.geo is not None else None,
place=tweet.place.full_name if tweet.place is not None else None,
insert_time=str(datetime.now()),
- poster=tweet.author.screen_name)
+ poster=tweet.author.screen_name, post_time=str(tweet.created_at), image_url=image_url_list)
ext.session.add(tweetDB)
ext.session.commit()
if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition_content.id).all():
@@ -94,50 +102,61 @@ def search_repo_conditions(repository_id):
if queryString != "":
for tweet in tw.Cursor(method=api.search, q=queryString).items(10):
tweetsFound.append(tweet)
- print(tweet.user.name + ' : ' + tweet.text + ' : ' + tweet.geo)
+ print(tweet.user.name + ' : ' + tweet.text + ' : ' + tweet.geo if tweet.geo is not None else '')
elif evaluation_mode == ConditionMode.all_and:
for tweet in tw.Cursor(method=api.search, q=queryString, geocode=coordinates_string).items(10):
tweetsFound.append(tweet)
print(tweet.user.name + ' : ' + tweet.text + ' : ' + str(tweet.geo))
for tweet in tweetsFound:
if not Tweet.query.filter_by(snowflake=str(tweet.id)).all():
+ image_url_list = ''
+ if 'media' in tweet.entities.keys():
+ for url in tweet.entities['media']:
+ image_url_list += (url['media_url_https'] + '|')
+ image_url_list = image_url_list[:-1]
+ else:
+ image_url_list = None
tweetDB = Tweet(snowflake=tweet.id, content=tweet.text,
location=tweet.geo['coordinates'] if tweet.geo is not None else None,
place=tweet.place.full_name if tweet.place is not None else None,
insert_time=str(datetime.now()),
- poster=tweet.author.screen_name)
+ poster=tweet.author.screen_name, post_time=str(tweet.created_at), image_url=image_url_list)
ext.session.add(tweetDB)
ext.session.commit()
if evaluation_mode == ConditionMode.all_or:
if ConditionType.hashtag in conditions_type.keys():
for condition_content in conditions_type[ConditionType.hashtag]:
if condition_content.content in [hashtag['text'] for hashtag in tweet.entities['hashtags']]:
- condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
- ext.session.add(condition_associated)
- ext.session.commit()
+ if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition_content.id).all():
+ condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
+ ext.session.add(condition_associated)
+ ext.session.commit()
if ConditionType.user in conditions_type.keys():
for condition_content in conditions_type[ConditionType.user]:
if condition_content.content == tweet.author.screen_name:
- condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
- ext.session.add(condition_associated)
- ext.session.commit()
+ if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition_content.id).all():
+ condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
+ ext.session.add(condition_associated)
+ ext.session.commit()
if ConditionType.time in conditions_type.keys():
for condition_content in conditions_type[ConditionType.time]:
condition_date_time = datetime.fromisoformat(condition_content.content[2:])
if condition_content.content[0] == '<':
if tweet.created_at < condition_date_time:
- condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
- ext.session.add(condition_associated)
- ext.session.commit()
+ if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition_content.id).all():
+ condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
+ ext.session.add(condition_associated)
+ ext.session.commit()
elif condition_content.content[0] == '>':
if tweet.created_at > condition_date_time:
- condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
- ext.session.add(condition_associated)
- ext.session.commit()
+ if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition_content.id).all():
+ condition_associated = Contains(cid=condition_content.id, snowflake=tweet.id)
+ ext.session.add(condition_associated)
+ ext.session.commit()
elif evaluation_mode == ConditionMode.all_and:
for condition in conditions:
- if Contains.query.filter_by(snowflake=str(tweet.id), cid=condition.id).all():
+ if not Contains.query.filter_by(snowflake=str(tweet.id), cid=condition.id).all():
condition_associated = Contains(cid=condition.id, snowflake=tweet.id)
ext.session.add(condition_associated)
ext.session.commit()