mirror of
https://github.com/pds-nest/nest.git
synced 2024-11-21 20:44:18 +00:00
Merge branch 'main' of zero.ryg.one:nest/g2-progetto
This commit is contained in:
FlaviaC-uni
commit
56939afb3b
16 changed files with 118 additions and 24 deletions
BIN
docs/build/doctrees/environment.pickle
vendored
BIN
docs/build/doctrees/environment.pickle
vendored
Binary file not shown.
|
|
@ -2,3 +2,4 @@
|
|||
====================================
|
||||
|
||||
.. automodule:: nest_crawler
|
||||
:imported-members:
|
||||
|
|
|
|||
5
docs/build/html/code/backend/index.html
vendored
5
docs/build/html/code/backend/index.html
vendored
|
|
@ -598,6 +598,11 @@ for example, any mapped columns or relationships.</p>
|
|||
<code class="sig-name descname"><span class="pre">is_active</span></code><a class="headerlink" href="#nest_backend.database.tables.Repository.is_active" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py attribute">
|
||||
<dt id="nest_backend.database.tables.Repository.is_deleted">
|
||||
<code class="sig-name descname"><span class="pre">is_deleted</span></code><a class="headerlink" href="#nest_backend.database.tables.Repository.is_deleted" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py attribute">
|
||||
<dt id="nest_backend.database.tables.Repository.evaluation_mode">
|
||||
<code class="sig-name descname"><span class="pre">evaluation_mode</span></code><a class="headerlink" href="#nest_backend.database.tables.Repository.evaluation_mode" title="Link a questa definizione">¶</a></dt>
|
||||
|
|
|
|||
35
docs/build/html/code/crawler/index.html
vendored
35
docs/build/html/code/crawler/index.html
vendored
|
|
@ -178,6 +178,41 @@
|
|||
|
||||
<div class="section" id="module-nest_crawler">
|
||||
<span id="nest-crawler-crawler-in-python"></span><h1><code class="docutils literal notranslate"><span class="pre">nest_crawler</span></code> - Crawler in Python<a class="headerlink" href="#module-nest_crawler" title="Link a questa intestazione">¶</a></h1>
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.associate_condition_tweet">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">associate_condition_tweet</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conditions_type</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">tweet</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.associate_condition_tweet" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.authenticate">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">authenticate</span></code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.authenticate" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.is_coordinate_inside_bounding_box">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">is_coordinate_inside_bounding_box</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">latitude</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">longitude</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">radius</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">tweet_latitude</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">tweet_longitude</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.is_coordinate_inside_bounding_box" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.is_repo_alert_triggered">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">is_repo_alert_triggered</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">repository_id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.is_repo_alert_triggered" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.search_repo_conditions">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">search_repo_conditions</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">repository_id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.search_repo_conditions" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.send_notification_email">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">send_notification_email</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">alert</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.send_notification_email" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt id="nest_crawler.send_notification_tweet">
|
||||
<code class="sig-prename descclassname"><span class="pre">nest_crawler.</span></code><code class="sig-name descname"><span class="pre">send_notification_tweet</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">alert</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#nest_crawler.send_notification_tweet" title="Link a questa definizione">¶</a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
|
|
|||
28
docs/build/html/genindex.html
vendored
28
docs/build/html/genindex.html
vendored
|
|
@ -256,8 +256,14 @@
|
|||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.OperationType.assign">assign (nest_backend.database.tables.OperationType attributo)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.gestione.authenticate">authenticate() (nel modulo nest_backend.gestione)</a>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.associate_condition_tweet">associate_condition_tweet() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.gestione.authenticate">authenticate() (nel modulo nest_backend.gestione)</a>
|
||||
|
||||
<ul>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.authenticate">(nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
</ul></li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Authorization">Authorization (classe in nest_backend.database.tables)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Repository.authorizations">authorizations (nest_backend.database.tables.Repository attributo)</a>
|
||||
|
|
@ -380,15 +386,21 @@
|
|||
<li><a href="code/backend/index.html#nest_backend.database.tables.Repository.id">(nest_backend.database.tables.Repository attributo)</a>
|
||||
</li>
|
||||
</ul></li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="code/backend/index.html#nest_backend.gestione.identity">identity() (nel modulo nest_backend.gestione)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Tweet.image_url">image_url (nest_backend.database.tables.Tweet attributo)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Tweet.insert_time">insert_time (nest_backend.database.tables.Tweet attributo)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Repository.is_active">is_active (nest_backend.database.tables.Repository attributo)</a>
|
||||
</li>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.is_coordinate_inside_bounding_box">is_coordinate_inside_bounding_box() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Repository.is_deleted">is_deleted (nest_backend.database.tables.Repository attributo)</a>
|
||||
</li>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.is_repo_alert_triggered">is_repo_alert_triggered() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.User.isAdmin">isAdmin (nest_backend.database.tables.User attributo)</a>
|
||||
</li>
|
||||
|
|
@ -586,6 +598,14 @@
|
|||
|
||||
<h2 id="S">S</h2>
|
||||
<table style="width: 100%" class="indextable genindextable"><tr>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.search_repo_conditions">search_repo_conditions() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.send_notification_email">send_notification_email() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
<li><a href="code/crawler/index.html#nest_crawler.send_notification_tweet">send_notification_tweet() (nel modulo nest_crawler)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Composed.snowflake">snowflake (nest_backend.database.tables.Composed attributo)</a>
|
||||
|
||||
|
|
@ -595,8 +615,6 @@
|
|||
<li><a href="code/backend/index.html#nest_backend.database.tables.Tweet.snowflake">(nest_backend.database.tables.Tweet attributo)</a>
|
||||
</li>
|
||||
</ul></li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="code/backend/index.html#nest_backend.database.tables.Repository.start">start (nest_backend.database.tables.Repository attributo)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
|
|
|
|||
BIN
docs/build/html/objects.inv
vendored
BIN
docs/build/html/objects.inv
vendored
Binary file not shown.
2
docs/build/html/searchindex.js
vendored
2
docs/build/html/searchindex.js
vendored
File diff suppressed because one or more lines are too long
|
|
@ -2,3 +2,4 @@
|
|||
====================================
|
||||
|
||||
.. automodule:: nest_crawler
|
||||
:imported-members:
|
||||
|
|
|
|||
|
|
@ -33,9 +33,16 @@ Lorenzo Balugani - DB & Backend Developer
|
|||
|
||||
| Data | Durata | Attività |
|
||||
|-------|--------|----------|
|
||||
| | | |
|
||||
|17/5 |4h |Bugfixing, supporto alla localizzazione degli errori |
|
||||
|18/5|2.0h|Bugfixing|
|
||||
|20/5|3.0h|API autorizzazioni, refactoring|
|
||||
|21/5|2.0h|Gestione tweet, rappresentazione tweet|
|
||||
|24/5|2.0h|Bugfixing|
|
||||
|25/5|4.0h|Docs, refactoring|
|
||||
|27/5|3.0h|Bugfixing|
|
||||
|28/5|6.0h|Bugfixing
|
||||
|-------|--------|
|
||||
|totale| __._h |
|
||||
|totale| 22.0h |
|
||||
|
||||
|
||||
Stefano Pigozzi - Sysadmin & Frontend Developer
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ from nest_backend.gestione import *
|
|||
import datetime
|
||||
from flask_cors import cross_origin
|
||||
from nest_backend.errors import *
|
||||
import nest_backend.app
|
||||
from nest_crawler.repo_search import search_repo_conditions
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
def tweet_importer(rid):
|
||||
import nest_backend.app
|
||||
with nest_backend.app.app.app_context():
|
||||
search_repo_conditions(rid)
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1,4 @@
|
|||
|
||||
from .alert_trigger import *
|
||||
from .associate_condition_tweet import *
|
||||
from .authentication import *
|
||||
from .repo_search import *
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from nest_backend.database import *
|
||||
from nest_backend.app import app, extension_sqlalchemy
|
||||
from nest_crawler.repo_search import search_repo_conditions
|
||||
from alert_trigger import is_repo_alert_triggered
|
||||
from nest_backend.app import app
|
||||
from .repo_search import search_repo_conditions
|
||||
from .alert_trigger import is_repo_alert_triggered
|
||||
|
||||
ext.init_app(app=app)
|
||||
|
||||
|
|
@ -12,8 +12,7 @@ def search_all_repo():
|
|||
search_repo_conditions(repo_id)
|
||||
is_repo_alert_triggered(repo_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with app.app_context():
|
||||
search_all_repo()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from datetime import datetime, timedelta
|
||||
from nest_backend.database import *
|
||||
from authentication import authenticate
|
||||
from .authentication import authenticate
|
||||
import smtplib
|
||||
|
||||
|
||||
def is_repo_alert_triggered(repository_id):
|
||||
repo = Repository.query.filter_by(id=repository_id).first()
|
||||
if repo is None:
|
||||
|
|
@ -56,6 +57,7 @@ def send_notification_email(alert):
|
|||
if smtpObj is not None:
|
||||
smtpObj.close()
|
||||
|
||||
|
||||
def send_notification_tweet(alert):
|
||||
api = authenticate()
|
||||
conditions_string = ''
|
||||
|
|
@ -65,3 +67,9 @@ def send_notification_tweet(alert):
|
|||
print(conditions_string)
|
||||
api.update_status(f"L'alert {alert.name} è stato attivato! C'è stato un incremento di popolarità negli argomenti di ricerca {conditions_string}")
|
||||
|
||||
|
||||
__all__ = (
|
||||
"is_repo_alert_triggered",
|
||||
"send_notification_email",
|
||||
"send_notification_tweet",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,3 +50,9 @@ def is_coordinate_inside_bounding_box(latitude, longitude, radius, tweet_latitud
|
|||
dLongitude = dLatitude * cos(radians(latitude))
|
||||
if (latitude - dLatitude < tweet_latitude < latitude+dLatitude) and (longitude-dLongitude < tweet_longitude < longitude+dLongitude):
|
||||
return True
|
||||
|
||||
|
||||
__all__ = (
|
||||
"associate_condition_tweet",
|
||||
"is_coordinate_inside_bounding_box",
|
||||
)
|
||||
|
|
@ -1,14 +1,20 @@
|
|||
import tweepy as tw
|
||||
import os
|
||||
|
||||
|
||||
def authenticate():
|
||||
c_k = "GEhtSyP9e98mzFeiOCSW0lvQX"
|
||||
c_s = "438cmYrl5xqaX2W7I2Bf5A9nF1pN5VtM9f77WYQnAXg1BwKJ27"
|
||||
a_t = "1380217745732689921-IW3U1JlxhnQeGBUrnHZ2nxbxhksXUZ"
|
||||
a_t_s = "EUoYNoj72rb2q00tUIW8eTcLJAhUAYPstZlV78W9cPpEJ"
|
||||
c_k = os.getenv('C_K')
|
||||
c_s = os.getenv('C_S')
|
||||
a_t = os.getenv('A_T')
|
||||
a_t_s = os.getenv('A_T_S')
|
||||
|
||||
auth = tw.OAuthHandler(c_k, c_s)
|
||||
auth.set_access_token(a_t, a_t_s)
|
||||
api = tw.API(auth, wait_on_rate_limit=True)
|
||||
# client = tw.Client(b_t, c_k, c_s, a_t, a_t_s, wait_on_rate_limit=True);
|
||||
return api
|
||||
return api
|
||||
|
||||
|
||||
__all__ = (
|
||||
"authenticate",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
from nest_backend.database import *
|
||||
import nest_crawler.authentication as authentication
|
||||
from datetime import datetime, timedelta
|
||||
from . import authentication
|
||||
from datetime import datetime
|
||||
import tweepy as tw
|
||||
from nest_crawler.associate_condition_tweet import associate_condition_tweet
|
||||
from .associate_condition_tweet import associate_condition_tweet
|
||||
|
||||
|
||||
def search_repo_conditions(repository_id):
|
||||
|
|
@ -142,3 +142,8 @@ def search_repo_conditions(repository_id):
|
|||
ext.session.add(composed)
|
||||
ext.session.commit()
|
||||
print(f"Done searching tweets from repo: {repo.name}")
|
||||
|
||||
|
||||
__all__ = (
|
||||
"search_repo_conditions",
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue