From 42ce7fa9954e71000c112d53901794918df974b6 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Thu, 28 Apr 2022 18:05:05 +0200 Subject: [PATCH] :tada: update some things --- .gitignore | 3 + .idea/runConfigurations/registered_fast.xml | 23 +++++++ .idea/runConfigurations/registered_slow.xml | 23 +++++++ docs/source/_extra/.gitignore | 0 docs/source/_static/.gitignore | 0 docs/source/cli.rst | 53 ++++++++++----- docs/source/index.rst | 3 +- docs/source/installation.rst | 71 +++++++++++++++++++++ docs/source/misc.rst | 16 +++++ docs/source/reference.rst | 2 + docs/source/trivia.rst | 7 -- io_beep_boop/api/client.py | 2 +- io_beep_boop/api/models.py | 2 - io_beep_boop/cli/__main__.py | 60 +++++++++++------ 14 files changed, 218 insertions(+), 47 deletions(-) create mode 100644 .idea/runConfigurations/registered_fast.xml create mode 100644 .idea/runConfigurations/registered_slow.xml create mode 100644 docs/source/_extra/.gitignore create mode 100644 docs/source/_static/.gitignore create mode 100644 docs/source/installation.rst create mode 100644 docs/source/misc.rst delete mode 100644 docs/source/trivia.rst diff --git a/.gitignore b/.gitignore index a2c3170..f689422 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ # Add your own ignores here! +input.txt +registered.txt +unregistered.txt ################## diff --git a/.idea/runConfigurations/registered_fast.xml b/.idea/runConfigurations/registered_fast.xml new file mode 100644 index 0000000..8321bd3 --- /dev/null +++ b/.idea/runConfigurations/registered_fast.xml @@ -0,0 +1,23 @@ + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/registered_slow.xml b/.idea/runConfigurations/registered_slow.xml new file mode 100644 index 0000000..08238e2 --- /dev/null +++ b/.idea/runConfigurations/registered_slow.xml @@ -0,0 +1,23 @@ + + + + + \ No newline at end of file diff --git a/docs/source/_extra/.gitignore b/docs/source/_extra/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/_static/.gitignore b/docs/source/_static/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 9f4b363..8cb7e7a 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -1,6 +1,6 @@ -###################### -Command-line interface -###################### +################################ +Using the command-line interface +################################ :mod:`io_beep_boop` includes a command line interface to (hopefully) facilitate the execution of certain tasks with the IO API. @@ -8,13 +8,13 @@ The interface can be invoked by entering the following in environments where the .. code-block:: console - $ io-beep-boop + (.venv)$ io-beep-boop All commands can be suffixed with ``--help`` to read their documentation: .. code-block:: console - $ io-beep-boop --help + (.venv)$ io-beep-boop --help Usage: io-beep-boop [OPTIONS] COMMAND [ARGS]... Options: @@ -37,12 +37,12 @@ API keys can be passed programmatically as the ``--token`` parameter, or manuall .. code-block:: console - $ io-beep-boop --token="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + (.venv)$ io-beep-boop --token="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" .. code-block:: console - $ io-beep-boop - Token: + (.venv)$ io-beep-boop + Token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Discover who is registered to a given service @@ -54,22 +54,43 @@ Given a text file containing a list of fiscal codes separated by newlines, :mod: Using the fast method --------------------- -.. todo:: - - Description of the fast method. +By using the :meth:`~io_beep_boop.api.client.get_subscriptions_on_day` method on all days in a given date range, :mod:`io_beep_boop` can determine the users who registered to the service in those days. .. code-block:: console - $ io-beep-boop registered-fast + (.venv)$ io-beep-boop registered-fast + +.. warning:: + + Internally, this uses the ``/subscription-feed/`` endpoint, which is not enabled by default for all API keys, and requires manual approval by the IO Team. + + If the token is not enabled to access the endpoint, a :class:`httpx.HTTPStatusError` will be raised. + + .. code-block:: console + + $ io-beep-boop registered-fast + ... + httpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://api.io.italia.it/api/v1/subscriptions-feed/2022-04-28' + For more information check: https://httpstatuses.com/403 Using the slow method --------------------- -.. todo:: - - Description of the fast method. +By using the :meth:`io_beep_boop.api.client.get_profile` methods on all profiles with the given fiscal codes, :mod:`io_beep_boop` can determine which ones of those users are registered to the service and which ones are not. .. code-block:: console - $ io-beep-boop registered-slow + (.venv)$ io-beep-boop registered-slow + +By default, the method performs a single HTTP request per second, in order to avoid rate limits; this can be changed with the ``--sleep`` option: + +.. code-block:: console + + (.venv)$ io-beep-boop registered-slow --sleep 5.0 + +.. warning:: + + This endpoint performs a HTTP request for every single fiscal code in your given input document, which works, but may not be allowed by IO App API Terms of Service due to the extreme amount of requests possibly generated. + + Try to keep the ``--sleep`` delay high! diff --git a/docs/source/index.rst b/docs/source/index.rst index ec05101..19bb4cc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,9 +7,10 @@ An experimental wrapper and command line interface for the Italian `IO App API < .. toctree:: + installation cli reference - trivia + misc Indices and tables diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..a743c35 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,71 @@ +############ +Installation +############ + +This page will act as a tutorial on how to safely setup a Python environment on a computer, and on how to install :mod:`io_beep_boop` inside. + +.. hint:: + + If you are a Python developer, and you intend to use this library to develop new tools, you might be interested to know that: + + - :mod:`io_beep_boop` is distributed through the `Python Package Index `_, so you can install it with any Python dependency manager and use it in any Python project; + - :mod:`io_beep_boop` supports :pep:`518`, so it may be installed from source using ``pip install .``; + - :mod:`io_beep_boop` uses `Poetry `_ as a dependency manager. + + +Installing Python +================= + +:mod:`io_beep_boop` requires Python 3.10 or later. + +You can download Python at the `Downloads page of the official website `_. + +Ensure that "Add Python to the PATH" is checked during installation, or you will not be able to run scripts from the command line! + + +Creating a :mod:`venv` +====================== + +To prevent dependency conflicts, it is highly suggested to create a new :mod:`venv` ( *v*\ irtual *env*\ ironment) to install :mod:`io_beep_boop` in. + +To do so, open a terminal or command prompt, create a new folder, access it, and when inside run ``python -m venv .venv``: + +.. code-block:: console + + $ cd Documents/Workspaces + $ mkdir IOTools + $ cd IOTools + $ python -m venv .venv + +Once created, run the following command to access the venv: + +.. code-block:: console + + $ source ./.venv/bin/activate + +.. code-block:: doscon + + > .\.venv\Scripts\activate + +You will have to activate the :mod:`venv` on every subsequent terminal session, or you won't be able to access the packages installed inside it! + +.. seealso:: + + `Installing packages using pip and virtual environments `_ by the Python Packaging Authority. + + +Installing :mod:`io_beep_boop` +============================== + +Once enabled the venv, you can install packages in it by using :mod:`pip`. + +Specifically, you'll want to install :mod:`io_beep_boop`: + +.. code-block:: console + + (.venv)$ pip install io-beep-boop + ... + Successfully installed io-beep-boop-0.1.0 + +The installation is complete! +You may proceed to :doc:`cli`. \ No newline at end of file diff --git a/docs/source/misc.rst b/docs/source/misc.rst new file mode 100644 index 0000000..341faec --- /dev/null +++ b/docs/source/misc.rst @@ -0,0 +1,16 @@ +############# +Miscellaneous +############# + +Inspired by +=========== + +The `ComuneDiRivoli/parlaConIO GitHub repository `_. + + +Package name +============ + +.. figure:: io-dota.webp + + The package name is a tribute to the Dota 2 hero of the same name, Io. diff --git a/docs/source/reference.rst b/docs/source/reference.rst index 7e4dadb..4d5f4fc 100644 --- a/docs/source/reference.rst +++ b/docs/source/reference.rst @@ -2,6 +2,8 @@ API Reference ############# +.. automodule:: io_beep_boop + :mod:`io_beep_boop.api` ======================= diff --git a/docs/source/trivia.rst b/docs/source/trivia.rst deleted file mode 100644 index ee74439..0000000 --- a/docs/source/trivia.rst +++ /dev/null @@ -1,7 +0,0 @@ -###### -Trivia -###### - -.. figure:: io-dota.webp - - The package name is a tribute to the Dota 2 hero of the same name, Io. \ No newline at end of file diff --git a/io_beep_boop/api/client.py b/io_beep_boop/api/client.py index 18cf505..6d3a4c4 100644 --- a/io_beep_boop/api/client.py +++ b/io_beep_boop/api/client.py @@ -122,7 +122,7 @@ class IOServiceClient(httpx.Client): https://developer.io.italia.it/openapi.html#operation/getSubscriptionsFeedForDate """ - return self.get(f"/subscriptions-feed/{date.isoformat()}") + return self.get(f"/subscriptions-feed/{date.strftime('%Y-%m-%d')}") # TODO: Service-level endpoints - since it seems weird to me that a service can spawn endless clones of itself? diff --git a/io_beep_boop/api/models.py b/io_beep_boop/api/models.py index 6db2491..d5a1aef 100644 --- a/io_beep_boop/api/models.py +++ b/io_beep_boop/api/models.py @@ -65,8 +65,6 @@ class GetMessageResponse(IOModel): class UserProfile(IOModel): - email: str - version: int sender_allowed: bool diff --git a/io_beep_boop/cli/__main__.py b/io_beep_boop/cli/__main__.py index d58dccc..213dd09 100644 --- a/io_beep_boop/cli/__main__.py +++ b/io_beep_boop/cli/__main__.py @@ -12,13 +12,15 @@ from ..api.models import SubscriptionsFeed def hash_fiscal_code(code: str) -> str: uppercased_code = code.upper() - hashed_code = hashlib.sha256(uppercased_code) + encoded_code = bytes(uppercased_code, encoding="utf8") + hashed_code = hashlib.sha256(encoded_code) hexed_hash = hashed_code.hexdigest() lowercased_hash = hexed_hash.lower() return lowercased_hash @click.group() +@click.version_option(package_name="io-beep-boop") @click.option( "-t", "--token", @@ -65,15 +67,17 @@ def main(ctx: click.Context, token: str, base_url: str): ) @click.option( "--start-date", - type=datetime.date, + type=click.DateTime(formats=["%Y-%m-%d"]), help="The date to start retrieving fiscal codes from.", + default=datetime.date.today().isoformat(), prompt=True, ) @click.option( "--end-date", - type=datetime.date, + type=click.DateTime(formats=["%Y-%m-%d"]), help="The date to stop retrieving fiscal codes at.", - default=datetime.date.today(), + default=datetime.date.today().isoformat(), + prompt=True, ) @click.option( "--sleep", @@ -99,12 +103,19 @@ def registered_fast(ctx: click.Context, input_file: t.TextIO, registered_file: t with click.progressbar(range(0, total_days), length=total_days, label="Retrieving data from the API...") as days: for day in days: - feed: SubscriptionsFeed = client.get_subscriptions_on_day(date=start_date + datetime.timedelta(days=day)) - - api_codes += set(feed.subscriptions) - api_codes -= set(feed.unsubscriptions) - - time.sleep(sleep) + while True: + try: + feed: SubscriptionsFeed = client.get_subscriptions_on_day(date=start_date + datetime.timedelta(days=day)) + except httpx.HTTPStatusError as e: + if e.response.status_code == 429: + continue + else: + raise + else: + api_codes += set(feed.subscriptions) + api_codes -= set(feed.unsubscriptions) + finally: + time.sleep(sleep) # Convert objects back to fiscal codes click.echo("Calculating registered fiscal codes...") @@ -162,17 +173,26 @@ def registered_slow(ctx: click.Context, input_file: t.TextIO, registered_file: t with click.progressbar(input_codes, label="Performing checks...") as codes: for code in codes: - try: - profile = client.get_profile(fiscal_code=code) - except httpx.HTTPStatusError: - unregistered_file.write(f"{code}\n") - else: - if not profile.sender_allowed: - unregistered_file.write(f"{code}\n") + while True: + try: + profile = client.get_profile(fiscal_code=code) + except httpx.HTTPStatusError as e: + if e.response.status_code == 429: + continue + elif e.response.status_code == 404: + unregistered_file.write(f"{code}\n") + break + else: + raise else: - registered_file.write(f"{code}\n") - finally: - time.sleep(sleep) + if not profile.sender_allowed: + unregistered_file.write(f"{code}\n") + break + else: + registered_file.write(f"{code}\n") + break + finally: + time.sleep(sleep) if __name__ == "__main__":