diff --git a/.vscode/launch.json b/.vscode/launch.json index 2190a32..a9cef56 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,7 @@ "type": "debugpy", "request": "launch", "module": "steamleaderboards", - "args": ["-o", "./data", "383980"] + "args": ["-o", "./data", "383980", "--limit", "10", "--request-delay", "1.5"] } ] } \ No newline at end of file diff --git a/steamleaderboards/__init__.py b/steamleaderboards/__init__.py index 34680bc..2ea2fe0 100644 --- a/steamleaderboards/__init__.py +++ b/steamleaderboards/__init__.py @@ -59,7 +59,7 @@ class ProtoLeaderboard: class Leaderboard: # noinspection PyMissingConstructor - def __init__(self, app_id=None, lbid=None, *, protoleaderboard=None, limit=5000, delay=None): + def __init__(self, app_id=None, lbid=None, *, protoleaderboard=None, limit=None, delay=None): if protoleaderboard: self.url = protoleaderboard.url self.lbid = protoleaderboard.lbid @@ -80,6 +80,8 @@ class Leaderboard: self.display_type = None else: raise ValueError("No app_id, lbid or protoleaderboard specified") + if limit is None: + limit = 5000 if delay is None: delay = 0.5 next_request_url = self.url @@ -89,12 +91,16 @@ class Leaderboard: _bs = BeautifulSoup(xml.content, features="lxml-xml") for entry in _bs.find_all("entry"): self.entries.append(Entry(entry)) - try: - next_request_url = _bs.find_all("nextRequestURL")[0].text - except IndexError: - next_request_url = None + if len(self.entries) >= limit: + next_request_url = None + break else: - time.sleep(delay) + try: + next_request_url = _bs.find_all("nextRequestURL")[0].text + except IndexError: + next_request_url = None + else: + time.sleep(delay) def __repr__(self): if self.name: diff --git a/steamleaderboards/__main__.py b/steamleaderboards/__main__.py index 12282ad..172a121 100644 --- a/steamleaderboards/__main__.py +++ b/steamleaderboards/__main__.py @@ -2,6 +2,7 @@ import argparse import importlib.metadata import pathlib import sys +import time from . import LeaderboardGroup, ProtoLeaderboard, Leaderboard, Entry @@ -11,7 +12,8 @@ parser = argparse.ArgumentParser( parser.add_argument("-o", "--output-dir", dest="output_dir", help="The directory where downloaded leaderboards should be stored in.", type=pathlib.Path) parser.add_argument("app_id", type=int, nargs="+") -parser.add_argument("-d", "--request-delay", dest="request_delay", help="How long to wait between two requests to the scoreboard API.", type=float) +parser.add_argument("-d", "--request-delay", dest="request_delay", help="How long to wait between two requests to the scoreboard API.", type=float, default=0.5) +parser.add_argument("-l", "--limit", dest="limit", help="How many entries to retrieve for each scoreboard.", type=int, default=5000) parser.add_argument("-V", "--version", action="version", version=importlib.metadata.version("steamleaderboards")) def main(): @@ -32,11 +34,13 @@ def main(): lg: LeaderboardGroup = LeaderboardGroup(app_id) for proto in lg.leaderboards: print(f"fetching full leaderboard: {app_id} {proto.name}", file=sys.stderr) - full: Leaderboard = proto.full(delay=args.request_delay) + full: Leaderboard = proto.full(limit=args.limit, delay=args.request_delay) with open(output_dir.joinpath(f"{full.app_id}_{full.name}.csv"), mode="w") as file: file.write(f"rank,steam_id,score,ugcid,details\n") for entry in full.entries: file.write(f"{entry.rank!r},{entry.steam_id!r},{entry.score!r},{entry.ugcid!r},{entry.details!r}\n") + print(f"done, resting for {args.request_delay} seconds", file=sys.stderr) + time.sleep(args.request_delay) if len(lg.leaderboards) == 0: print(f"game has no leaderboards", file=sys.stderr)