unimore/bda-6-steffo
unimore/bda-6-steffo
1
Fork 0
mirror of https://github.com/Steffo99/unimore-bda-6.git synced 2024-11-21 23:44:19 +00:00
Code Activity

Some memory usage tweaks

Browse source
This commit is contained in:
Steffo 2023-02-18 03:18:34 +01:00
parent 35616d35c7
commit 61141248db
Signed by: steffo
GPG key ID: 2A24051445686895
3 changed files with 69 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.vscode/launch.json vendored
View file

@ -12,7 +12,6 @@
"justMyCode": false, "justMyCode": false,
"env": { "env": {
"NLTK_DATA": "./data/nltk", "NLTK_DATA": "./data/nltk",
"DATA_SET_SIZE": "250",
"XLA_FLAGS": "--xla_gpu_cuda_data_dir=/opt/cuda" "XLA_FLAGS": "--xla_gpu_cuda_data_dir=/opt/cuda"
}, },
"cwd": "${workspaceFolder}", "cwd": "${workspaceFolder}",

12
unimore_bda_6/__main__.py
View file

@ -31,8 +31,6 @@ def main():
log.fatal("MongoDB database is not available, exiting...") log.fatal("MongoDB database is not available, exiting...")
exit(1) exit(1)
reviews = reviews_collection(db)
for sample_func in [ for sample_func in [
sample_reviews_polar, sample_reviews_polar,
sample_reviews_varied, sample_reviews_varied,
@ -94,7 +92,13 @@ def main():
slog.warning("%s is not supported by %s, skipping run...", SentimentAnalyzer.__name__, Tokenizer.__name__) slog.warning("%s is not supported by %s, skipping run...", SentimentAnalyzer.__name__, Tokenizer.__name__)
break break
with Caches.from_database_samples(collection=reviews, sample_func=sample_func) as datasets:
with mongo_client_from_config() as db:
reviews = reviews_collection(db)
datasets_cm = Caches.from_database_samples(collection=reviews, sample_func=sample_func)
datasets = datasets_cm.__enter__()
try:
try: try:
slog.info("Training sentiment analyzer: %s", sa) slog.info("Training sentiment analyzer: %s", sa)
sa.train(training_dataset_func=datasets.training, validation_dataset_func=datasets.validation) sa.train(training_dataset_func=datasets.training, validation_dataset_func=datasets.validation)
@ -111,6 +115,8 @@ def main():
successful_runs += 1 successful_runs += 1
cumulative_evaluation_results += evaluation_results cumulative_evaluation_results += evaluation_results
break break
finally:
datasets_cm.__exit__()
slog.info("Cumulative evaluation results: %s", cumulative_evaluation_results) slog.info("Cumulative evaluation results: %s", cumulative_evaluation_results)

2
unimore_bda_6/database/connection.py
View file

@ -23,7 +23,7 @@ def mongo_client_from_config() -> t.ContextManager[pymongo.MongoClient]:
yield client yield client
log.info("Closing connection to MongoDB...") log.debug("Closing connection to MongoDB...")
client.close() client.close()
log.debug("Closed connection to MongoDB!") log.debug("Closed connection to MongoDB!")