Bump version to 1.1.0

As mentioned in #1.

.gitignore

@@ -1,3 +1,5 @@
+data/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.vscode/launch.json

@@ -0,0 +1,12 @@
+{
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"name": "Fetch Rivals of Aether scoreboards",
+			"type": "debugpy",
+			"request": "launch",
+			"module": "steamleaderboards",
+			"args": ["-o", "./data", "383980", "--limit", "10", "--request-delay", "1.5"]
+		}
+	]
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "steamleaderboards"
-version = "1.0.0"
+version = "1.1.0"
 description = "Retrieve and parse Steam leaderboards"
 authors = ["Stefano Pigozzi <me@steffo.eu>"]
 maintainers = ["Stefano Pigozzi <me@steffo.eu>"]

steamleaderboards/__init__.py

@@ -1,6 +1,7 @@
 import requests
 from bs4 import BeautifulSoup
 import typing
+import time
 
 
 class LeaderboardGroup:
@@ -52,13 +53,13 @@ class ProtoLeaderboard:
         self.display_type = int(soup.displaytype.text)
         self.app_id = app_id
 
-    def full(self) -> "Leaderboard":
-        return Leaderboard(protoleaderboard=self)
+    def full(self, *args, **kwargs) -> "Leaderboard":
+        return Leaderboard(*args, **kwargs, protoleaderboard=self)
 
 
 class Leaderboard:
     # noinspection PyMissingConstructor
-    def __init__(self, app_id=None, lbid=None, *, protoleaderboard=None):
+    def __init__(self, app_id=None, lbid=None, *, protoleaderboard=None, limit=None, delay=None):
         if protoleaderboard:
             self.url = protoleaderboard.url
             self.lbid = protoleaderboard.lbid
@@ -79,6 +80,10 @@ class Leaderboard:
             self.display_type = None
         else:
             raise ValueError("No app_id, lbid or protoleaderboard specified")
+        if limit is None:
+            limit = 5000
+        if delay is None:
+            delay = 0.5
         next_request_url = self.url
         self.entries = []
         while next_request_url:
@@ -86,14 +91,16 @@ class Leaderboard:
             _bs = BeautifulSoup(xml.content, features="lxml-xml")
             for entry in _bs.find_all("entry"):
                 self.entries.append(Entry(entry))
-            if _bs.response.entryend:
-                entry_end = int(_bs.response.entryend.text)
-                if entry_end < int(_bs.response.totalleaderboardentries.text):
-                    next_request_url = f"https://steamcommunity.com/stats/{self.app_id}/leaderboards/{self.lbid}/?xml=1&start={entry_end + 1}"
-                else:
+                if len(self.entries) >= limit:
                     next_request_url = None
+                    break
             else:
-                next_request_url = None
+                try:
+                    next_request_url = _bs.find_all("nextRequestURL")[0].text
+                except IndexError:
+                    next_request_url = None
+                else:
+                    time.sleep(delay)
 
     def __repr__(self):
         if self.name:

steamleaderboards/__main__.py

@@ -2,6 +2,7 @@ import argparse
 import importlib.metadata
 import pathlib
 import sys
+import time
 from . import LeaderboardGroup, ProtoLeaderboard, Leaderboard, Entry
 
 
@@ -11,6 +12,8 @@ parser = argparse.ArgumentParser(
 
 parser.add_argument("-o", "--output-dir", dest="output_dir", help="The directory where downloaded leaderboards should be stored in.", type=pathlib.Path)
 parser.add_argument("app_id", type=int, nargs="+")
+parser.add_argument("-d", "--request-delay", dest="request_delay", help="How long to wait between two requests to the scoreboard API.", type=float, default=0.5)
+parser.add_argument("-l", "--limit", dest="limit", help="How many entries to retrieve for each scoreboard.", type=int, default=5000)
 parser.add_argument("-V", "--version", action="version", version=importlib.metadata.version("steamleaderboards"))
 
 def main():
@@ -31,11 +34,13 @@ def main():
 		lg: LeaderboardGroup = LeaderboardGroup(app_id)
 		for proto in lg.leaderboards:
 			print(f"fetching full leaderboard: {app_id} {proto.name}", file=sys.stderr)
-			full: Leaderboard = proto.full()
+			full: Leaderboard = proto.full(limit=args.limit, delay=args.request_delay)
 			with open(output_dir.joinpath(f"{full.app_id}_{full.name}.csv"), mode="w") as file:
 				file.write(f"rank,steam_id,score,ugcid,details\n")
 				for entry in full.entries:
 					file.write(f"{entry.rank!r},{entry.steam_id!r},{entry.score!r},{entry.ugcid!r},{entry.details!r}\n")
+			print(f"done, resting for {args.request_delay} seconds", file=sys.stderr)
+			time.sleep(args.request_delay)
 		if len(lg.leaderboards) == 0:
 			print(f"game has no leaderboards", file=sys.stderr)