refactor(steam): replace certain API calls to avoid rate limit (#20)

buff2steam/__main__.py

@@ -6,10 +6,6 @@
 from buff2steam.provider.steam import Steam
 
 
-def remove_exponent(d):
-    return d.quantize(decimal.Decimal(1)) if d == d.to_integral() else d.normalize()
-
-
 async def main_loop(buff, steam):
     total_page = await buff.get_total_page()
     visited = set()
@@ -22,44 +18,49 @@ async def main_loop(buff, steam):
                 continue
 
             market_hash_name = item['market_hash_name']
-            buff_min_price = remove_exponent(decimal.Decimal(item['sell_min_price']) * 100)
-            buff_says_steam_price = remove_exponent(decimal.Decimal(item['goods_info']['steam_price_cny']) * 100)
+            buff_min_price = int(decimal.Decimal(item['sell_min_price']) * 100)
+            buff_says_steam_price = int(decimal.Decimal(item['goods_info']['steam_price_cny']) * 100)
 
             if not config['main']['max_price'] > buff_min_price > config['main']['min_price']:
                 logger.debug(f'{market_hash_name}: buff_min_price({buff_min_price}) not in range, skipping...')
                 continue
 
             buff_says_ratio = buff_min_price / buff_says_steam_price if buff_says_steam_price else 1
-            accept_buff_threshold = decimal.Decimal(config['main']['accept_buff_threshold'])
+            accept_buff_threshold = config['main']['accept_buff_threshold']
             if buff_says_ratio > accept_buff_threshold:
                 logger.debug(
                     f'{market_hash_name}: {buff_says_ratio} > {accept_buff_threshold}, skipping...'
                 )
                 continue
 
             logger.debug(f'Processing {market_hash_name}...')
-            listings_data = await steam.listings_data(market_hash_name)
-            current_ratio = buff_min_price / listings_data['converted_price']
-            orders_data = await steam.orders_data(market_hash_name)
-            highest_buy_order = decimal.Decimal(orders_data['highest_buy_order'])
-            wanted_cnt = orders_data['wanted_cnt']
-            steam_tax_ratio = decimal.Decimal(listings_data['steam_tax_ratio'])
-            highest_buy_order_ratio = buff_min_price / (highest_buy_order * steam_tax_ratio)
+
+            price_overview_data = await steam.price_overview_data(market_hash_name)
+            if price_overview_data is None:
+                continue
+
+            volume = price_overview_data['volume']
+            orders_data = None
+            if volume > 0:
+                orders_data = await steam.orders_data(market_hash_name)
+
+            current_ratio = buff_min_price / (price_overview_data['price'] / (1 + steam.fee_rate))
             buff_min_price_human = float(buff_min_price / 100)
 
+            result = [
+                f'buff_id/price: {item["id"]}/{buff_min_price_human};',
+            ]
+
+            if orders_data is None:
+                result.append(f'volume: {volume}; ratio: {current_ratio:04.2f}')
+            else:
+                b_o_ratio = buff_min_price / (orders_data['highest_buy_order'] / (1 + steam.fee_rate))
+                result.append(f'sell/want: {orders_data["sell_order_count"]}/{orders_data["buy_order_count"]};')
+                result.append(f'volume: {volume}; b_o_ratio: {b_o_ratio:04.2f}; ratio: {current_ratio:04.2f}')
+
             visited.add(item['id'])
 
-            logger.info(' '.join([
-                'buff_id/price: {buff_id}/{buff_price};'.format(
-                    buff_id=item['id'], buff_price=buff_min_price_human
-                ),
-                'sell/want: {sell}/{want};'.format(
-                    sell=listings_data['total_count'], want=wanted_cnt
-                ),
-                'b_o_ratio: {b_o_ratio:04.2f}; ratio: {ratio:04.2f}'.format(
-                    b_o_ratio=highest_buy_order_ratio, ratio=current_ratio
-                )
-            ]))
+            logger.info(' '.join(result))
 
 
 async def main():

buff2steam/exceptions.py

@@ -8,3 +8,7 @@ class SteamAPI429Error(SteamError):
 
 class SteamItemNameIdNotFoundError(SteamError):
     pass
+
+
+class BuffError(Exception):
+    pass

buff2steam/provider/buff.py

@@ -5,6 +5,7 @@
 import httpx
 
 from buff2steam import logger
+from buff2steam.exceptions import BuffError
 
 
 class Buff:
@@ -51,7 +52,7 @@ async def request(self, *args, **kwargs) -> dict:
 
             response = await self.opener.request(*args, **kwargs)
             if response.json()['code'] != 'OK':
-                raise Exception(response.json())
+                raise BuffError(response.json())
 
             return response.json()['data']
 

buff2steam/provider/steam.py

@@ -1,4 +1,5 @@
 import asyncio
+import decimal
 import re
 import time
 
@@ -13,19 +14,19 @@
 
 class Steam:
     base_url = 'https://steamcommunity.com'
+    fee_rate = 0.15
 
     web_sell = '/market/sellitem'
     web_inventory = '/inventory/{steam_id}/{game_appid}/{context_id}'
     web_listings = '/market/listings/{game_appid}/{market_hash_name}'
-    web_listings_render = web_listings + '/render'
     web_item_orders_histogram = '/market/itemordershistogram'
 
     item_nameid_pattern = re.compile(r'Market_LoadOrderSpread\(\s*(\d+)\s*\)')
-    wanted_cnt_pattern = re.compile(r'<span\s*class="market_commodity_orders_header_promote">(\d+)</span>')
+    currency_pattern = re.compile(r'[^\d.]')
 
     def __init__(
             self, asf_config=None, steam_id=None, game_appid='', context_id=2,
-            request_interval=30, request_kwargs=None
+            request_interval=4, request_kwargs=None
     ):
         self.request_interval = request_interval
         self.request_locks = {}  # {caller: [asyncio.Lock, last_request_time]}
@@ -39,7 +40,6 @@ def __init__(
             context_id=context_id
         )
         self.web_listings = self.web_listings.replace('{game_appid}', game_appid)
-        self.web_listings_render = self.web_listings_render.replace('{game_appid}', game_appid)
 
     async def __aenter__(self):
         return self
@@ -67,36 +67,37 @@ async def _request(self, *args, **kwargs):
 
             return res
 
-    @tenacity.retry(retry=tenacity.retry_if_exception_type(buff2steam.exceptions.SteamAPI429Error))
-    async def _web_listings_render(self, market_hash_name):
-        return await self._request('GET', self.web_listings_render.format(market_hash_name=market_hash_name), params={
-            'count': 1,
-            'currency': 23
-        })
-
     @tenacity.retry(retry=tenacity.retry_if_exception_type(buff2steam.exceptions.SteamAPI429Error))
     async def _web_listings(self, market_hash_name):
         return await self._request('GET', self.web_listings.format(market_hash_name=market_hash_name))
 
-    @tenacity.retry(retry=tenacity.retry_if_exception_type(buff2steam.exceptions.SteamAPI429Error))
     async def _item_orders_histogram(self, item_nameid):
         return await self._request('GET', self.web_item_orders_histogram, params={
             'language': 'schinese',
             'currency': 23,
             'item_nameid': item_nameid,
+            'norender': 1,
+            'country': 'CN',
+            'two_factor': 0
+        })
+
+    async def _price_overview(self, market_hash_name):
+        return await self._request('GET', '/market/priceoverview', params={
+            'appid': self.game_appid,
+            'market_hash_name': market_hash_name,
+            'currency': 23,
         })
 
-    async def listings_data(self, market_hash_name):
-        res = (await self._web_listings_render(market_hash_name)).json()
+    async def price_overview_data(self, market_hash_name):
+        res = (await self._price_overview(market_hash_name)).json()
 
-        listinginfo = res['listinginfo'][next(iter(res['listinginfo']))]
-        converted_price = listinginfo['converted_price']
-        converted_fee = listinginfo['converted_fee']
+        if 'lowest_price' not in res:
+            logger.debug(f'{market_hash_name}: {res}')
+            return None
 
         return {
-            'converted_price': converted_price,
-            'total_count': res['total_count'],
-            'steam_tax_ratio': converted_price / (converted_price + converted_fee)
+            'price': int(decimal.Decimal(self.currency_pattern.sub('', res['lowest_price'])) * 100),
+            'volume': int(res.get('volume', '0').replace(',', '')),
         }
 
     async def orders_data(self, market_hash_name):
@@ -107,11 +108,17 @@ async def orders_data(self, market_hash_name):
             logger.critical(res.text)
             raise buff2steam.exceptions.SteamItemNameIdNotFoundError()
 
-        res = await self._item_orders_histogram(item_nameid[0])
+        try:
+            res = await self._item_orders_histogram(item_nameid[0])
+        except buff2steam.exceptions.SteamAPI429Error:
+            logger.debug(f'SteamAPI429Error({market_hash_name}): Could not get orders data, skipping...')
+            return None
 
         orders_data = res.json()
 
         return {
-            'highest_buy_order': orders_data['highest_buy_order'],
-            'wanted_cnt': self.wanted_cnt_pattern.findall(orders_data['buy_order_summary'])[0]
+            'sell_order_count': int(orders_data['sell_order_count'].replace(',', '')),
+            'buy_order_count': int(orders_data['buy_order_count'].replace(',', '')),
+            'lowest_sell_order': int(orders_data['lowest_sell_order']),
+            'highest_buy_order': int(orders_data['highest_buy_order']),
         }

config.sample.json

@@ -19,7 +19,7 @@
     }
   },
   "steam": {
-    "request_interval": 30,
+    "request_interval": 4,
     "requests_kwargs": {
       "headers": {
         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"

setup.py

@@ -5,17 +5,17 @@
 
 setup(
     name='buff2steam',
-    version='0.3.1',
+    version='0.4.0',
     python_requires='>=3.7',
     url='https://github.com/hldh214/buff2steam',
     license='Unlicense',
     description='Yet another steam trade bot w/ buff.163.com',
     long_description=long_description,
     packages=['buff2steam'],
     install_requires=[
-        'httpx',
-        'loguru',
-        'tenacity'
+        'httpx==0.*',
+        'loguru==0.*',
+        'tenacity==8.*'
     ],
     author='Jim',
     author_email='hldh214@gmail.com',